diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index 6a517f3d0..ea8262423 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -1,5 +1,8 @@ # SPDX-License-Identifier: AGPL-3.0-or-later -"""Startpage's language & region selectors are a mess .. +"""Startpage requires solving an Anubis POW captcha (difficulty 4). +Solving it requires a lot of CPU, so the engine is set inactive by default. + +Startpage's language & region selectors are a mess .. .. _startpage regions: @@ -84,6 +87,7 @@ Startpage's category (for Web-search, News, Videos, ..) is set by """ # pylint: disable=too-many-statements +import hashlib import re import typing as t from collections import OrderedDict @@ -99,7 +103,7 @@ from searx.enginelib import EngineCache from searx.enginelib.traits import EngineTraits from searx.exceptions import SearxEngineCaptchaException from searx.locales import region_tag -from searx.network import get # see https://github.com/searxng/searxng/issues/762 +from searx.network import get, post # see https://github.com/searxng/searxng/issues/762 from searx.utils import ( eval_xpath, extr, @@ -176,6 +180,45 @@ def setup(_: dict[str, t.Any]) -> bool | None: sc_code_cache_sec = 3600 """Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`.""" +# startpage's anubis difficulty is set to 4 +max_difficulty = 4 + + +def _solve_anubis(resp) -> str: + """Anubis POW solver""" + payload = loads(extr(resp.text, '")) + challenge = payload["challenge"] + difficulty = int(payload["rules"]["difficulty"]) + if difficulty > max_difficulty: + raise SearxEngineCaptchaException(message="startpage: Anubis difficulty too high") + prefix = "0" * difficulty + blob = challenge["randomData"].encode() + for nonce in range(16**difficulty * 8): # max search is 8x average search, e^-8 = 0.034% will fail + digest = hashlib.sha256(blob + str(nonce).encode()).hexdigest() + if digest.startswith(prefix): + break + else: + raise SearxEngineCaptchaException(message="startpage: Anubis failed") + + pass_resp = get( + f"{base_url}/.within.website/x/cmd/anubis/api/pass-challenge", + params={ + "id": challenge["id"], + "response": digest, + "nonce": nonce, + "redir": str(resp.url), + "elapsedTime": "1", + }, + cookies=resp.cookies, + allow_redirects=False, + ) + auth = pass_resp.cookies.get("spchal-auth") + if not auth: + raise SearxEngineCaptchaException(message="startpage: Anubis pass-challenge failed") + auth = str(auth) + CACHE.set("SPCHAL_AUTH", auth, expire=240) + return auth + def get_sc_code(params): """Get an actual ``sc`` argument from Startpage's search form (HTML page). @@ -201,6 +244,9 @@ def get_sc_code(params): logger.debug("get_sc_code: request headers: %s", headers) resp = get(get_sc_url, headers=headers) + if 'id="anubis_challenge"' in resp.text: + resp = get(get_sc_url, headers=headers, cookies={"spchal-auth": _solve_anubis(resp)}) + # ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers) # ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg # ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21 @@ -239,8 +285,8 @@ def request(query, params): Additionally the arguments form Startpage's search form needs to be set in HTML POST data / compare ```` elements: :py:obj:`search_form_xpath`. """ - engine_region = traits.get_region(params["searxng_locale"], "en-US") - engine_language = traits.get_language(params["searxng_locale"], "en") + engine_region = traits.get_region(params["searxng_locale"], "en_US") + engine_language = traits.get_language(params["searxng_locale"], "english") params["headers"]["Origin"] = base_url params["headers"]["Referer"] = base_url + "/" @@ -262,9 +308,9 @@ def request(query, params): args["language"] = engine_language args["lui"] = engine_language + args["segment"] = "startpage.udog" if params["pageno"] > 1: args["page"] = params["pageno"] - args["segment"] = "startpage.udog" # Build cookie lang_homepage = "en" @@ -289,6 +335,8 @@ def request(query, params): cookie["search_results_region"] = engine_region params["cookies"]["preferences"] = "N1N".join(["%sEEE%s" % x for x in cookie.items()]) + if auth := CACHE.get("SPCHAL_AUTH"): + params["cookies"]["spchal-auth"] = auth logger.debug("cookie preferences: %s", params["cookies"]["preferences"]) logger.debug("data: %s", args) @@ -400,6 +448,18 @@ def _get_image_result(result) -> dict[str, t.Any] | None: def response(resp): + if 'id="anubis_challenge"' in resp.text: + params = resp.search_params + params["cookies"]["spchal-auth"] = _solve_anubis(resp) + resp = post( + params["url"] or search_url, + data=params["data"], + headers=params["headers"], + cookies=params["cookies"], + ) + if 'id="anubis_challenge"' in resp.text: + raise SearxEngineCaptchaException() + categ = startpage_categ.capitalize() results_raw = "{" + extr(resp.text, f"React.createElement(UIStartpage.AppSerp{categ}, {{", "}})") + "}}" diff --git a/searx/settings.yml b/searx/settings.yml index ceddec985..3f7268b04 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -2343,18 +2343,21 @@ engines: shortcut: sp startpage_categ: web categories: [general, web] + inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669 - name: startpage news engine: startpage startpage_categ: news categories: [news, web] shortcut: spn + inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669 - name: startpage images engine: startpage startpage_categ: images categories: [images, web] shortcut: spi + inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669 - name: steam engine: steam