[upd] web-client (simple): Bump the minor group (#6692)

Bumps the minor group in /client/simple with 2 updates: [@biomejs/biome](https://github.com/biomejs/biome/tree/HEAD/packages/@biomejs/biome) and [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node).


Updates `@biomejs/biome` from 2.5.11 to 2.5.12
- [Release notes](https://github.com/biomejs/biome/releases)
- [Changelog](https://github.com/biomejs/biome/blob/main/packages/@biomejs/biome/CHANGELOG.md)
- [Commits](https://github.com/biomejs/biome/commits/@biomejs/biome@2.5.12/packages/@biomejs/biome)

Updates `@types/node` from 26.4.1 to 26.5.0
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)

---
updated-dependencies:
- dependency-name: "@biomejs/biome"
  dependency-version: 2.5.12
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: minor
- dependency-name: "@types/node"
  dependency-version: 26.5.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

client/simple/package-lock.json

@@ -15,8 +15,8 @@
         "swiped-events": "1.2.0"
       },
       "devDependencies": {
-        "@biomejs/biome": "2.5.11",
+        "@biomejs/biome": "2.5.12",
-        "@types/node": "^26.4.0",
+        "@types/node": "^26.5.0",
         "browserslist": "^4.28.8",
         "browserslist-to-esbuild": "^2.1.1",
         "edge.js": "^6.5.1",
@@ -69,9 +69,9 @@
       }
     },
     "node_modules/@biomejs/biome": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.12.tgz",
-      "integrity": "sha512-Tj0dnkLPdW0ASjHfj2D/ZkkvPU2wrFmnE1jWTD2xzV1ycapV1DutbYXk4NDnR3rYTi1ZCbNFD4G2gRMEY65WaA==",
+      "integrity": "sha512-Lw4VHZRebrReBBnlHa12JQjnIBm3JJAA55PDB9LbBVBF0q4RYphm6KfmIjqtPhf61MxZ5Q9KoK8R8x+7per5Aw==",
       "dev": true,
       "license": "MIT OR Apache-2.0",
       "bin": {
@@ -85,20 +85,20 @@
         "url": "https://opencollective.com/biome"
       },
       "optionalDependencies": {
-        "@biomejs/cli-darwin-arm64": "2.5.11",
+        "@biomejs/cli-darwin-arm64": "2.5.12",
-        "@biomejs/cli-darwin-x64": "2.5.11",
+        "@biomejs/cli-darwin-x64": "2.5.12",
-        "@biomejs/cli-linux-arm64": "2.5.11",
+        "@biomejs/cli-linux-arm64": "2.5.12",
-        "@biomejs/cli-linux-arm64-musl": "2.5.11",
+        "@biomejs/cli-linux-arm64-musl": "2.5.12",
-        "@biomejs/cli-linux-x64": "2.5.11",
+        "@biomejs/cli-linux-x64": "2.5.12",
-        "@biomejs/cli-linux-x64-musl": "2.5.11",
+        "@biomejs/cli-linux-x64-musl": "2.5.12",
-        "@biomejs/cli-win32-arm64": "2.5.11",
+        "@biomejs/cli-win32-arm64": "2.5.12",
-        "@biomejs/cli-win32-x64": "2.5.11"
+        "@biomejs/cli-win32-x64": "2.5.12"
       }
     },
     "node_modules/@biomejs/cli-darwin-arm64": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.12.tgz",
-      "integrity": "sha512-6SGZxoKbXvUjMn1t6A98HqWISPnGNbYs0R/Rt2JarmXBSev+lva4QxUMWEBX9lX1Wo1XTJ78uk5xVDtG58SRZg==",
+      "integrity": "sha512-lCRY1rwgNeWNgTr4DI/u6ZwXTRwRLHAvbaio1YLLGS+4r1nhvB2ssyPqIpfUSmRveNfv0fn/N58C7CAdK2XVrg==",
       "cpu": [
         "arm64"
       ],
@@ -113,9 +113,9 @@
       }
     },
     "node_modules/@biomejs/cli-darwin-x64": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.12.tgz",
-      "integrity": "sha512-nYkXY7tLBEgnGbYapDKAyKzgt44ZEyG+AKalvTXtCWKYgepI9dw327q+cVgedxm+Udi1ZzHKUyZrIusHi/KQbw==",
+      "integrity": "sha512-vhPgwnh+6tN3ArdAXuET99xaNbFt7CG82Bqn+omHVLC5xdVx45JsYjGPmUIGNzjDek5XdNCP1HKksK7fn8+3bQ==",
       "cpu": [
         "x64"
       ],
@@ -130,9 +130,9 @@
       }
     },
     "node_modules/@biomejs/cli-linux-arm64": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.12.tgz",
-      "integrity": "sha512-3PVLSTD9RR73rvVPt5G3T1gc+ycggWEGfTD7RvzzbtcDPD27NxgxBbAFfpm7DXJKW6VLHWE1lLMGvFt2Qxjcow==",
+      "integrity": "sha512-2gp8aVwXYKdAtmBfRFCUuyDMcfN1ahHqUkGfLYrZlNRFmryMATLVvJgWKvyA8wu4Rwn5OSxM1UcUmOuOFNGeBQ==",
       "cpu": [
         "arm64"
       ],
@@ -150,9 +150,9 @@
       }
     },
     "node_modules/@biomejs/cli-linux-arm64-musl": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.12.tgz",
-      "integrity": "sha512-qhyZUMyCbWYFV2bAwRNVvfMVZ+hv7WYl6mossGrxC+uiQQXhvsuWWU8zz6jYX0mChZd9MgQZbm4vozTmG/5iGw==",
+      "integrity": "sha512-couHYjFLL5uuI8ne6zhT7KwEsXo5YP7ry/2xmEqah7qanu0YmfDi3mwJg47YXSuv/NpZj22CZzcRH/5c4gjPSQ==",
       "cpu": [
         "arm64"
       ],
@@ -170,9 +170,9 @@
       }
     },
     "node_modules/@biomejs/cli-linux-x64": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.12.tgz",
-      "integrity": "sha512-JOytptlsgM33B2MMFUg8iBrb4IKpbD5JnJrSeYiaFEeAj4vuXx0iQSQZ4qK7sqyMtfjZxxPdNdMZZVL4y/mFyA==",
+      "integrity": "sha512-SnvOs3TSTiuia4SQOUNe1aWC9RT4+YkjcKnOhL/nsKOV0k5ycgBkDzF0lUxKn1V7Q8CLTRq6iV23ZAivHomRoA==",
       "cpu": [
         "x64"
       ],
@@ -190,9 +190,9 @@
       }
     },
     "node_modules/@biomejs/cli-linux-x64-musl": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.12.tgz",
-      "integrity": "sha512-oRRlrchG5EfrEL/EmtT1qUjSNHk3/5LGeZhQqADBBAJF1b1ET6964xEKe7aGlGARzDfza8H/seEsFJl7S6Ql9w==",
+      "integrity": "sha512-8A0oDW58/w9f/PQNYuq0sGUZtGtGrkNF4Z6n0PUoXpLCshi85vtKTv1XSznQawhdE4MXJ8ufpzHXyLFe87M/+w==",
       "cpu": [
         "x64"
       ],
@@ -210,9 +210,9 @@
       }
     },
     "node_modules/@biomejs/cli-win32-arm64": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.12.tgz",
-      "integrity": "sha512-e49E6K9hzH/ohJNx8Y26mY8HaV4I4ZViIeoqhKsmoXLKHhQnMeBAVqCgsGf2Wa3lXlS7RkporDXMHHWkzvZzFw==",
+      "integrity": "sha512-b9vtoZFsuZt1pdjNwJvXl0f+BpayRzV008uS2+JpmwIKdSE2qdu4A/l04FESwLoou5g2E/Qlec0xwJydZplH+A==",
       "cpu": [
         "arm64"
       ],
@@ -227,9 +227,9 @@
       }
     },
     "node_modules/@biomejs/cli-win32-x64": {
-      "version": "2.5.11",
+      "version": "2.5.12",
-      "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.11.tgz",
+      "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.12.tgz",
-      "integrity": "sha512-QSQr/KjOgXA7OzXJUWS+oguKyAZ3Q0l/lnlDGbu397eKo83atuWUjBPJrsqbKNF6CARGw8XXJLGzpHC8Ryhd4Q==",
+      "integrity": "sha512-B1R/l+CwEpKFSuqiwePzPNRk1EiJN8kc0UhdafNz6MZN9v5OFP9HYP1irptvWzHrwVI4blVNGMbxc5zt70m3IA==",
       "cpu": [
         "x64"
       ],
@@ -1601,13 +1601,13 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "26.4.1",
+      "version": "26.5.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.0.tgz",
-      "integrity": "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA==",
+      "integrity": "sha512-dVSGpriSoCgz8WnDNTuSSuSv1PC/ALXihO4ulRZt7Md8k9mlbdin3lGOcDE8SnWOgf513ByWlXd7BK4azmyg/A==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
-        "undici-types": "~8.3.0"
+        "undici-types": "~8.9.0"
       }
     },
     "node_modules/@types/pluralize": {
@@ -5004,9 +5004,9 @@
       }
     },
     "node_modules/undici-types": {
-      "version": "8.3.0",
+      "version": "8.9.0",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
-      "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
+      "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
       "dev": true,
       "license": "MIT"
     },

client/simple/package.json

@@ -29,8 +29,8 @@
     "swiped-events": "1.2.0"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.5.11",
+    "@biomejs/biome": "2.5.12",
-    "@types/node": "^26.4.0",
+    "@types/node": "^26.5.0",
     "browserslist": "^4.28.8",
     "browserslist-to-esbuild": "^2.1.1",
     "edge.js": "^6.5.1",

docs/dev/engines/online/europepmc.rst

@@ -0,0 +1,8 @@
+.. _europepmc engine:
+
+==========
+Europe PMC
+==========
+
+.. automodule:: searx.engines.europepmc
+   :members:

searx/engines/bing.py

@@ -72,43 +72,21 @@ def get_locale_params(engine_region: str | None) -> dict[str, str] | None:
     return {"mkt": engine_region}
 
 
-def override_accept_language(params: "OnlineParams", engine_region: str | None) -> None:
-    """Override the ``Accept-Language`` header.
-
-    The default header built by :py:class:`~searx.search.processors.online.OnlineProcessor`
-    appends ``en;q=0.3`` as a fallback language::
-
-        Accept-Language: de,de-DE;q=0.7,en;q=0.3
-
-    Bing seems to better select the results locale based on the
-    ``Accept-Language`` value header.
-
-    This function is shared with :py:mod:`searx.engines.bing_images`,
-    :py:mod:`searx.engines.bing_news`, and :py:mod:`searx.engines.bing_videos`.
-    """
-
-    if not engine_region or engine_region == "clear":
-        return
-
-    lang = engine_region.split("-")[0]
-    params["headers"]["Accept-Language"] = f"{engine_region},{lang};q=0.9"
-
-
 def request(query: str, params: "OnlineParams"):
     """Assemble a Bing-Web request."""
 
     engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
 
-    override_accept_language(params, engine_region)
-
     query_params: dict[str, str | int] = {
         "q": query,
         "adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
     }
 
-    locale_params = get_locale_params(engine_region)
+    if engine_region and engine_region != "clear":
-    if locale_params:
+        lang, _, cc = engine_region.partition("-")
-        query_params.update(locale_params)
+        query_params["setlang"] = lang
+        if cc and cc not in ("us", "cn", "ru"):  # bing just sends junk for these
+            query_params["cc"] = cc
 
     params["url"] = f"{base_url}/search?{urlencode(query_params)}"
 

searx/engines/bing_images.py

@@ -9,7 +9,6 @@ from lxml import html
 from searx.engines.bing import (  # pylint: disable=unused-import
     fetch_traits,
     get_locale_params,
-    override_accept_language,
 )
 
 # about
@@ -44,8 +43,6 @@ def request(query, params):
 
     engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
 
-    override_accept_language(params, engine_region)
-
     # build URL query
     # - example: https://www.bing.com/images/async?q=foo&async=1&first=1&count=35
     query_params = {

searx/engines/bing_news.py

@@ -12,10 +12,7 @@ from urllib.parse import urlencode
 from lxml import html
 
 from searx.enginelib.traits import EngineTraits
-from searx.engines.bing import (
+from searx.engines.bing import get_locale_params
-    get_locale_params,
-    override_accept_language,
-)
 from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
 
 # about
@@ -54,8 +51,6 @@ def request(query, params):
 
     engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
 
-    override_accept_language(params, engine_region)
-
     # build URL query
     # - example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
     page = int(params.get("pageno", 1)) - 1

searx/engines/bing_videos.py

@@ -9,7 +9,6 @@ from lxml import html
 from searx.engines.bing import (  # pylint: disable=unused-import
     fetch_traits,
     get_locale_params,
-    override_accept_language,
 )
 from searx.engines.bing_images import time_map
 from searx.utils import eval_xpath, eval_xpath_getindex
@@ -39,8 +38,6 @@ def request(query, params):
 
     engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
 
-    override_accept_language(params, engine_region)
-
     # build URL query
     # - example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
     query_params = {

searx/engines/core.py

@@ -141,12 +141,13 @@ def response(resp: "SXNG_Response") -> EngineResults:
             if name:
                 authors.add(name)
 
+        tag = result.get("fieldOfStudy")
         res.add(
             res.types.Paper(
                 title=result.get("title"),
                 url=url,
                 content=result.get("fullText", "") or "",
-                tags=result.get("fieldOfStudy", []),
+                tags=[tag] if tag else [],
                 publishedDate=published_date,
                 type=result.get("documentType", "") or "",
                 authors=authors,

searx/engines/duckduckgo_web.py

@@ -14,11 +14,12 @@ can't build it ourselves and must scrape it from the HTML pages.
 """
 
 import typing as t
+import re
 
-from urllib.parse import quote_plus
+from urllib.parse import quote_plus, urljoin
 from lxml import html
 
-from searx.utils import html_to_text, gen_useragent, extract_text, eval_xpath
+from searx.utils import html_to_text, extract_text, eval_xpath
 from searx.result_types import EngineResults
 from searx.enginelib import EngineCache
 from searx.network import get
@@ -38,7 +39,6 @@ about = {
 # engine dependent config
 categories = ["general"]
 paging = True
-_HTTP_User_Agent: str = gen_useragent()
 
 base_url = "https://duckduckgo.com"
 
@@ -73,6 +73,8 @@ def _fetch_first_page_link(
     resp = get(
         url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
         headers=headers,
+        impersonate="firefox",
+        default_headers=False,
         timeout=2,
     )
 
@@ -96,6 +98,43 @@ def _cache_key(query: str, pageno: int) -> str:
     return f"nextpage_url|{query}|{pageno}"
 
 
+def _solve_jsa(resp: "SXNG_Response") -> "SXNG_Response":
+    """Duckduckgo sometimes issues a challenge instead of json."""
+
+    # length that a real browser would report for where the broken snippet is
+    html_len = {
+        "<p><div></p><p></div": 32,
+        "<li><div></li><li></div": 29,
+        "<div><div></div><div></div": 33,
+        "<br><div></br><br></div": 23,
+    }
+
+    js = resp.text or ""
+    jsa_match = re.search(r"let jsa = (\d+);.*?DDG\.deep\.initialize\('([^']+)'", js, re.S)
+    if not jsa_match:
+        return resp
+
+    js_functions = dict(re.findall(r"let (\w+) = function\(num\) \{([^}]*)\};", js))
+    jsa = int(jsa_match.group(1))
+    try:
+        for name in re.findall(r"jsa = (\w+)\(jsa\);", js):
+            body = js_functions[name]
+            mul = re.search(r"num \* (\d+)", body)
+            jsa = jsa * int(mul.group(1)) if mul else jsa + html_len[re.search(r"`([^`]+)`", body).group(1)]
+    except (KeyError, AttributeError):
+        return resp
+
+    params = resp.search_params
+    follow = get(
+        urljoin("https://links.duckduckgo.com", jsa_match.group(2) + str(jsa)),
+        headers=params["headers"],
+        impersonate="firefox",
+        default_headers=False,
+    )
+    follow.search_params = params
+    return follow
+
+
 def request(query: str, params: "OnlineParams") -> None:
 
     if len(query) >= 500:
@@ -103,25 +142,15 @@ def request(query: str, params: "OnlineParams") -> None:
         params["url"] = None
         return
 
-    headers = params["headers"]
+    # firefox TLS only
-
+    params["impersonate"] = "firefox"
-    # The vqd value is generated from the query and the UA header.  To be able
+    params["default_headers"] = False
-    # to reuse the vqd value, the UA header must be static.
-    headers["User-Agent"] = _HTTP_User_Agent
-    headers["Accept"] = "*/*"
-    headers["Referer"] = f"{base_url}/"
-    headers["Host"] = "duckduckgo.com"
-
-    # Sec-Fetch headers are required to not get blocked when sending a Firefox user agent
-    headers["Sec-Fetch-Dest"] = "script"
-    headers["Sec-Fetch-Mode"] = "no-cors"
-    headers["Sec-Fetch-Site"] = "same-site"
 
     api_url = ""
     if params["pageno"] > 1:
         api_url = CACHE.get(_cache_key(query, params["pageno"]))
     else:
-        api_url = _fetch_first_page_link(query, headers)
+        api_url = _fetch_first_page_link(query, params["headers"])
 
     if not api_url:
         params["url"] = None
@@ -129,14 +158,27 @@ def request(query: str, params: "OnlineParams") -> None:
 
     params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
 
+    # loads as a script
+    headers = params["headers"]
+    headers["Accept"] = "*/*"
+    headers["Sec-Fetch-Dest"] = "script"
+    headers["Sec-Fetch-Mode"] = "no-cors"
+    headers["Sec-Fetch-Site"] = "same-site"
+    headers["Referer"] = f"{base_url}/"
+
     # TODO: support safesearch, timerange and engine traits  # pylint:disable=fixme
 
 
 def response(resp: "SXNG_Response"):
     res = EngineResults()
-    res_json = resp.json()
 
-    for result in res_json["results"]:
+    # check if ddg returns a challenge
+    # e.g. 'site:github.com searxng'
+    if "let jsa =" in (resp.text or ""):
+        resp = _solve_jsa(resp)
+
+    results = resp.json()["results"]
+    for result in results:
         if "u" not in result:
             continue
 
@@ -144,13 +186,13 @@ def response(resp: "SXNG_Response"):
             res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
         )
 
-    # link to next page
+    if results:
-    next_page_path = res_json["results"][-1].get("n")
+        next_page_path = results[-1].get("n")
-    if next_page_path:
+        if next_page_path:
-        CACHE.set(
+            CACHE.set(
-            _cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
+                _cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
-            base_url + next_page_path,
+                base_url + next_page_path,
-            expire=60 * 60,
+                expire=60 * 60,
-        )
+            )
 
     return res

searx/engines/europepmc.py

@@ -0,0 +1,150 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""`Europe PMC`_ provides comprehensive access to life sciences literature from
+trusted sources.  With Europe PMC you can search and read millions of
+publications, preprints and other documents enriched with links to supporting
+data, reviews, protocols, and other relevant resources.
+
+.. _Europe PMC: https://europepmc.org/
+
+Configuration
+=============
+
+.. code:: yaml
+
+   - name: europepmc
+     engine: europepmc
+     shortcut: epmc
+
+Implementations
+===============
+
+"""
+
+import typing as t
+
+from datetime import datetime
+from urllib.parse import urlencode
+
+from dateutil.parser import isoparse
+
+from searx.enginelib import EngineCache
+from searx.result_types import EngineResults
+from searx.utils import html_to_text
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
+
+
+about = {
+    "website": "https://europepmc.org/",
+    "wikidata_id": "Q5412157",
+    "official_api_documentation": "https://europepmc.org/RestfulWebService",
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": "JSON",
+}
+
+categories = ["science", "scientific publications"]
+paging = True
+
+# engine dependent config
+search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
+article_url = "https://europepmc.org/article/"
+
+page_size = 20
+
+CACHE: EngineCache
+"""Cache for storing the pagination cursor."""
+
+
+def setup(engine_settings: dict[str, t.Any]):
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])
+
+
+def _cache_key(query: str, page: int) -> str:
+    return f"{query}|{page}"
+
+
+def request(query: str, params: "OnlineParams") -> None:
+    args = {
+        "query": query,
+        "format": "json",
+        "resultType": "core",
+        "pageSize": page_size,
+    }
+
+    if params["pageno"] > 1:
+        if cursor := CACHE.get(_cache_key(query, params["pageno"])):
+            args["cursorMark"] = cursor
+        else:
+            # no cached cursor for that page
+            params["url"] = None
+            return
+
+    params["url"] = f"{search_url}?{urlencode(args)}"
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
+
+    json_resp = resp.json()
+
+    # store pagination cursor for loading next pages in cache
+    if next_cursor := json_resp.get("nextCursorMark"):
+        next_page = resp.search_params["pageno"] + 1
+        query = resp.search_params["query"]
+        CACHE.set(_cache_key(query, next_page), next_cursor)
+
+    all_results = json_resp.get("resultList", {}).get("result", [])
+
+    for item in all_results:
+        source = item.get("source", "")
+        identifier = item.get("id", "")
+        url = f"{article_url}{source}/{identifier}" if source and identifier else ""
+
+        journal_info: dict[str, t.Any] = item.get("journalInfo", {})
+        journal: dict[str, t.Any] = journal_info.get("journal", {})
+
+        res.add(
+            res.types.Paper(
+                url=url,
+                title=html_to_text(item.get("title", "")),
+                content=html_to_text(item.get("abstractText", "")),
+                journal=journal.get("title", ""),
+                issn=[journal.get("issn", "")],
+                authors=_get_authors(item),
+                doi=item.get("doi", ""),
+                publishedDate=_get_published_date(item),
+                type=", ".join((item.get("pubTypeList", {})).get("pubType", [])),
+                pdf_url=_get_pdf_url(item),
+                html_url=url,
+            )
+        )
+
+    return res
+
+
+def _get_authors(item: dict[str, t.Any]) -> list:
+    """Extract the list of authors from the item."""
+    if authors := item.get("authorString", None):
+        authors = [author.strip().rstrip(".") for author in authors.split(",") if author.strip()]
+    else:
+        authors = []
+    return authors
+
+
+def _get_pdf_url(item: dict[str, t.Any]) -> str:
+    """Extract the PDF URL in case it is open access."""
+    for url_info in (item.get("fullTextUrlList", {})).get("fullTextUrl", []):
+        if url_info.get("documentStyle") == "pdf" and url_info.get("availabilityCode") == "OA":
+            return url_info.get("url", "")
+    return ""
+
+
+def _get_published_date(item: dict[str, t.Any]) -> datetime | None:
+    """Extract the published date from the item and convert it to a datetime object."""
+    if unformatted_date := item.get("firstPublicationDate"):
+        return isoparse(unformatted_date)
+    return None

searx/engines/pinterest.py

@@ -2,72 +2,85 @@
 """Pinterest (images)"""
 
 from json import dumps
+import typing as t
+from searx.result_types import EngineResults
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
 
 about = {
-    "website": 'https://www.pinterest.com/',
+    "website": "https://www.pinterest.com/",
-    "wikidata_id": 'Q255381',
+    "wikidata_id": "Q255381",
-    "official_api_documentation": 'https://developers.pinterest.com/docs/api/v5/',
+    "official_api_documentation": "https://developers.pinterest.com/docs/api/v5/",
     "use_official_api": False,
     "require_api_key": False,
-    "results": 'JSON',
+    "results": "JSON",
 }
 
-categories = ['images']
+categories = ["images"]
 paging = True
 
-base_url = 'https://www.pinterest.com'
+base_url = "https://www.pinterest.com"
 
 
-def request(query, params):
+def request(query: str, params: "OnlineParams") -> None:
+
     args = {
-        'options': {
+        "options": {
-            'query': query,
+            "query": query,
-            'bookmarks': [params['engine_data'].get('bookmark', '')],
+            "bookmarks": [params["engine_data"].get("bookmark", "")],
         },
-        'context': {},
+        "context": {},
     }
-    params['url'] = f"{base_url}/resource/BaseSearchResource/get/?data={dumps(args)}"
+    params["url"] = f"{base_url}/resource/BaseSearchResource/get/?data={dumps(args)}"
-    params['headers'] = {
+    params["headers"] = {
-        'X-Pinterest-AppState': 'active',
+        "X-Requested-With": "XMLHttpRequest",
-        'X-Pinterest-Source-Url': '/ideas/',
+        "X-Pinterest-AppState": "active",
-        'X-Pinterest-PWS-Handler': 'www/ideas.js',
+        "X-Pinterest-Source-Url": "/ideas/",
+        "X-Pinterest-PWS-Handler": "www/ideas.js",
     }
 
-    return params
 
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
+    json_resp: dict[str, t.Any] = resp.json()  # type: ignore
 
-def response(resp):
+    res.add(
-    results = []
-
-    json_resp = resp.json()
-
-    results.append(
         {
-            'engine_data': json_resp['resource_response']['bookmark'],
+            "engine_data": json_resp["resource_response"]["bookmark"],
             # it's called bookmark by pinterest, but it's rather a nextpage
             # parameter to get the next results
-            'key': 'bookmark',
+            "key": "bookmark",
         }
     )
 
-    for result in json_resp['resource_response']['data']['results']:
+    for result in json_resp["resource_response"]["data"]["results"]:
 
-        if result['type'] == 'story':
+        if result["type"] == "story":
             continue
 
-        main_image = result['images']['orig']
+        main_image = result["images"]["orig"]
-        results.append(
+
-            {
+        title = result.get("title") or result.get("grid_title") or ""
-                'template': 'images.html',
+        if len(title) < 5:
-                'url': result.get('link') or f"{base_url}/pin/{result['id']}/",
+            visual_annotation = result.get("pin_join", {}).get("visual_annotation")
-                'title': result.get('title') or result.get('grid_title'),
+            if visual_annotation:
-                'content': (result.get('rich_summary') or {}).get('display_description') or "",
+                title = visual_annotation[0]
-                'img_src': main_image['url'],
+            else:
-                'thumbnail_src': result['images']['236x']['url'],
+                title = result.get("name") or result.get("auto_alt_text") or ""
-                'source': (result.get('rich_summary') or {}).get('site_name'),
+
-                'resolution': f"{main_image['width']}x{main_image['height']}",
+        res.add(
-                'author': f"{result['pinner'].get('full_name')} ({result['pinner']['username']})",
+            res.types.Image(
-            }
+                url=result.get("link") or f"{base_url}/pin/{result['id']}/",
+                title=title,
+                content=(result.get("rich_summary") or {}).get("display_description") or "",
+                img_src=main_image["url"],
+                thumbnail_src=result["images"]["236x"]["url"],
+                source=(result.get("rich_summary") or {}).get("site_name") or "",
+                resolution=f"{main_image['width']}x{main_image['height']}",
+                author=f"{result['pinner'].get('full_name')} ({result['pinner']['username']})",
+            )
         )
 
-    return results
+    return res

searx/engines/startpage.py

@@ -1,5 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Startpage's language & region selectors are a mess ..
+"""Startpage requires solving an Anubis POW captcha (difficulty 4).
+Solving it requires a lot of CPU, so the engine is set inactive by default.
+
+Startpage's language & region selectors are a mess ..
 
 .. _startpage regions:
 
@@ -84,6 +87,7 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
 """
 # pylint: disable=too-many-statements
 
+import hashlib
 import re
 import typing as t
 from collections import OrderedDict
@@ -99,7 +103,7 @@ from searx.enginelib import EngineCache
 from searx.enginelib.traits import EngineTraits
 from searx.exceptions import SearxEngineCaptchaException
 from searx.locales import region_tag
-from searx.network import get  # see https://github.com/searxng/searxng/issues/762
+from searx.network import get, post  # see https://github.com/searxng/searxng/issues/762
 from searx.utils import (
     eval_xpath,
     extr,
@@ -176,6 +180,45 @@ def setup(_: dict[str, t.Any]) -> bool | None:
 sc_code_cache_sec = 3600
 """Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
 
+# startpage's anubis difficulty is set to 4
+max_difficulty = 4
+
+
+def _solve_anubis(resp) -> str:
+    """Anubis POW solver"""
+    payload = loads(extr(resp.text, '<script id="anubis_challenge" type="application/json">', "</script>"))
+    challenge = payload["challenge"]
+    difficulty = int(payload["rules"]["difficulty"])
+    if difficulty > max_difficulty:
+        raise SearxEngineCaptchaException(message="startpage: Anubis difficulty too high")
+    prefix = "0" * difficulty
+    blob = challenge["randomData"].encode()
+    for nonce in range(16**difficulty * 8):  # max search is 8x average search, e^-8 = 0.034% will fail
+        digest = hashlib.sha256(blob + str(nonce).encode()).hexdigest()
+        if digest.startswith(prefix):
+            break
+    else:
+        raise SearxEngineCaptchaException(message="startpage: Anubis failed")
+
+    pass_resp = get(
+        f"{base_url}/.within.website/x/cmd/anubis/api/pass-challenge",
+        params={
+            "id": challenge["id"],
+            "response": digest,
+            "nonce": nonce,
+            "redir": str(resp.url),
+            "elapsedTime": "1",
+        },
+        cookies=resp.cookies,
+        allow_redirects=False,
+    )
+    auth = pass_resp.cookies.get("spchal-auth")
+    if not auth:
+        raise SearxEngineCaptchaException(message="startpage: Anubis pass-challenge failed")
+    auth = str(auth)
+    CACHE.set("SPCHAL_AUTH", auth, expire=240)
+    return auth
+
 
 def get_sc_code(params):
     """Get an actual ``sc`` argument from Startpage's search form (HTML page).
@@ -201,6 +244,9 @@ def get_sc_code(params):
     logger.debug("get_sc_code: request headers: %s", headers)
     resp = get(get_sc_url, headers=headers)
 
+    if 'id="anubis_challenge"' in resp.text:
+        resp = get(get_sc_url, headers=headers, cookies={"spchal-auth": _solve_anubis(resp)})
+
     # ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)
     # ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg
     # ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21
@@ -239,8 +285,8 @@ def request(query, params):
     Additionally the arguments form Startpage's search form needs to be set in
     HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.
     """
-    engine_region = traits.get_region(params["searxng_locale"], "en-US")
+    engine_region = traits.get_region(params["searxng_locale"], "en_US")
-    engine_language = traits.get_language(params["searxng_locale"], "en")
+    engine_language = traits.get_language(params["searxng_locale"], "english")
 
     params["headers"]["Origin"] = base_url
     params["headers"]["Referer"] = base_url + "/"
@@ -262,9 +308,9 @@ def request(query, params):
         args["language"] = engine_language
         args["lui"] = engine_language
 
+    args["segment"] = "startpage.udog"
     if params["pageno"] > 1:
         args["page"] = params["pageno"]
-        args["segment"] = "startpage.udog"
 
     # Build cookie
     lang_homepage = "en"
@@ -289,6 +335,8 @@ def request(query, params):
         cookie["search_results_region"] = engine_region
 
     params["cookies"]["preferences"] = "N1N".join(["%sEEE%s" % x for x in cookie.items()])
+    if auth := CACHE.get("SPCHAL_AUTH"):
+        params["cookies"]["spchal-auth"] = auth
     logger.debug("cookie preferences: %s", params["cookies"]["preferences"])
 
     logger.debug("data: %s", args)
@@ -400,6 +448,18 @@ def _get_image_result(result) -> dict[str, t.Any] | None:
 
 
 def response(resp):
+    if 'id="anubis_challenge"' in resp.text:
+        params = resp.search_params
+        params["cookies"]["spchal-auth"] = _solve_anubis(resp)
+        resp = post(
+            params["url"] or search_url,
+            data=params["data"],
+            headers=params["headers"],
+            cookies=params["cookies"],
+        )
+        if 'id="anubis_challenge"' in resp.text:
+            raise SearxEngineCaptchaException()
+
     categ = startpage_categ.capitalize()
     results_raw = "{" + extr(resp.text, f"React.createElement(UIStartpage.AppSerp{categ}, {{", "}})") + "}}"
 

searx/search/processors/online.py

@@ -88,6 +88,9 @@ class HTTPParams(t.TypedDict):
     impersonate: t.NotRequired[str]
     """curl_cffi impersonate target. Default: ``chrome``."""
 
+    default_headers: t.NotRequired[bool]
+    """Ask curl_cffi to skip setting the impersonated browser's default headers."""
+
     curl_options: t.NotRequired[dict[int, t.Any]]
     """Any extra libcurl options for the request."""
 
@@ -169,8 +172,8 @@ class OnlineProcessor(EngineProcessor):
             "cookies": params["cookies"],
             "auth": params["auth"],
         }
-        for key in ("curl_options", "impersonate"):
+        for key in ("curl_options", "impersonate", "default_headers"):
-            if params.get(key):
+            if params.get(key) is not None:
                 request_args[key] = params[key]
 
         verify = params.get("verify")

searx/settings.yml

@@ -792,6 +792,10 @@ engines:
       require_api_key: false
       results: JSON
 
+  - name: europepmc
+    engine: europepmc
+    shortcut: epmc
+
   - name: erowid
     engine: xpath
     paging: true
@@ -2339,18 +2343,21 @@ engines:
     shortcut: sp
     startpage_categ: web
     categories: [general, web]
+    inactive: true  # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
 
   - name: startpage news
     engine: startpage
     startpage_categ: news
     categories: [news, web]
     shortcut: spn
+    inactive: true  # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
 
   - name: startpage images
     engine: startpage
     startpage_categ: images
     categories: [images, web]
     shortcut: spi
+    inactive: true  # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
 
   - name: steam
     engine: steam