diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py
index 6a517f3d0..ea8262423 100644
--- a/searx/engines/startpage.py
+++ b/searx/engines/startpage.py
@@ -1,5 +1,8 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
-"""Startpage's language & region selectors are a mess ..
+"""Startpage requires solving an Anubis POW captcha (difficulty 4).
+Solving it requires a lot of CPU, so the engine is set inactive by default.
+
+Startpage's language & region selectors are a mess ..
.. _startpage regions:
@@ -84,6 +87,7 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
"""
# pylint: disable=too-many-statements
+import hashlib
import re
import typing as t
from collections import OrderedDict
@@ -99,7 +103,7 @@ from searx.enginelib import EngineCache
from searx.enginelib.traits import EngineTraits
from searx.exceptions import SearxEngineCaptchaException
from searx.locales import region_tag
-from searx.network import get # see https://github.com/searxng/searxng/issues/762
+from searx.network import get, post # see https://github.com/searxng/searxng/issues/762
from searx.utils import (
eval_xpath,
extr,
@@ -176,6 +180,45 @@ def setup(_: dict[str, t.Any]) -> bool | None:
sc_code_cache_sec = 3600
"""Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
+# startpage's anubis difficulty is set to 4
+max_difficulty = 4
+
+
+def _solve_anubis(resp) -> str:
+ """Anubis POW solver"""
+ payload = loads(extr(resp.text, '"))
+ challenge = payload["challenge"]
+ difficulty = int(payload["rules"]["difficulty"])
+ if difficulty > max_difficulty:
+ raise SearxEngineCaptchaException(message="startpage: Anubis difficulty too high")
+ prefix = "0" * difficulty
+ blob = challenge["randomData"].encode()
+ for nonce in range(16**difficulty * 8): # max search is 8x average search, e^-8 = 0.034% will fail
+ digest = hashlib.sha256(blob + str(nonce).encode()).hexdigest()
+ if digest.startswith(prefix):
+ break
+ else:
+ raise SearxEngineCaptchaException(message="startpage: Anubis failed")
+
+ pass_resp = get(
+ f"{base_url}/.within.website/x/cmd/anubis/api/pass-challenge",
+ params={
+ "id": challenge["id"],
+ "response": digest,
+ "nonce": nonce,
+ "redir": str(resp.url),
+ "elapsedTime": "1",
+ },
+ cookies=resp.cookies,
+ allow_redirects=False,
+ )
+ auth = pass_resp.cookies.get("spchal-auth")
+ if not auth:
+ raise SearxEngineCaptchaException(message="startpage: Anubis pass-challenge failed")
+ auth = str(auth)
+ CACHE.set("SPCHAL_AUTH", auth, expire=240)
+ return auth
+
def get_sc_code(params):
"""Get an actual ``sc`` argument from Startpage's search form (HTML page).
@@ -201,6 +244,9 @@ def get_sc_code(params):
logger.debug("get_sc_code: request headers: %s", headers)
resp = get(get_sc_url, headers=headers)
+ if 'id="anubis_challenge"' in resp.text:
+ resp = get(get_sc_url, headers=headers, cookies={"spchal-auth": _solve_anubis(resp)})
+
# ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)
# ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg
# ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21
@@ -239,8 +285,8 @@ def request(query, params):
Additionally the arguments form Startpage's search form needs to be set in
HTML POST data / compare ```` elements: :py:obj:`search_form_xpath`.
"""
- engine_region = traits.get_region(params["searxng_locale"], "en-US")
- engine_language = traits.get_language(params["searxng_locale"], "en")
+ engine_region = traits.get_region(params["searxng_locale"], "en_US")
+ engine_language = traits.get_language(params["searxng_locale"], "english")
params["headers"]["Origin"] = base_url
params["headers"]["Referer"] = base_url + "/"
@@ -262,9 +308,9 @@ def request(query, params):
args["language"] = engine_language
args["lui"] = engine_language
+ args["segment"] = "startpage.udog"
if params["pageno"] > 1:
args["page"] = params["pageno"]
- args["segment"] = "startpage.udog"
# Build cookie
lang_homepage = "en"
@@ -289,6 +335,8 @@ def request(query, params):
cookie["search_results_region"] = engine_region
params["cookies"]["preferences"] = "N1N".join(["%sEEE%s" % x for x in cookie.items()])
+ if auth := CACHE.get("SPCHAL_AUTH"):
+ params["cookies"]["spchal-auth"] = auth
logger.debug("cookie preferences: %s", params["cookies"]["preferences"])
logger.debug("data: %s", args)
@@ -400,6 +448,18 @@ def _get_image_result(result) -> dict[str, t.Any] | None:
def response(resp):
+ if 'id="anubis_challenge"' in resp.text:
+ params = resp.search_params
+ params["cookies"]["spchal-auth"] = _solve_anubis(resp)
+ resp = post(
+ params["url"] or search_url,
+ data=params["data"],
+ headers=params["headers"],
+ cookies=params["cookies"],
+ )
+ if 'id="anubis_challenge"' in resp.text:
+ raise SearxEngineCaptchaException()
+
categ = startpage_categ.capitalize()
results_raw = "{" + extr(resp.text, f"React.createElement(UIStartpage.AppSerp{categ}, {{", "}})") + "}}"
diff --git a/searx/settings.yml b/searx/settings.yml
index ceddec985..3f7268b04 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -2343,18 +2343,21 @@ engines:
shortcut: sp
startpage_categ: web
categories: [general, web]
+ inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
- name: startpage news
engine: startpage
startpage_categ: news
categories: [news, web]
shortcut: spn
+ inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
- name: startpage images
engine: startpage
startpage_categ: images
categories: [images, web]
shortcut: spi
+ inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
- name: steam
engine: steam