[upd] web-client (simple): Bump the minor group (#6692)

Bumps the minor group in /client/simple with 2 updates: [@biomejs/biome](https://github.com/biomejs/biome/tree/HEAD/packages/@biomejs/biome) and [@types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node).


Updates `@biomejs/biome` from 2.5.11 to 2.5.12
- [Release notes](https://github.com/biomejs/biome/releases)
- [Changelog](https://github.com/biomejs/biome/blob/main/packages/@biomejs/biome/CHANGELOG.md)
- [Commits](https://github.com/biomejs/biome/commits/@biomejs/biome@2.5.12/packages/@biomejs/biome)

Updates `@types/node` from 26.4.1 to 26.5.0
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/node)

---
updated-dependencies:
- dependency-name: "@biomejs/biome"
  dependency-version: 2.5.12
  dependency-type: direct:development
  update-type: version-update:semver-patch
  dependency-group: minor
- dependency-name: "@types/node"
  dependency-version: 26.5.0
  dependency-type: direct:development
  update-type: version-update:semver-minor
  dependency-group: minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/container.yml

@@ -62,7 +62,7 @@ jobs:
           python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Setup QEMU
-        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8  # v4.2.0
+        uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a  # v4.3.0
 
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
@@ -105,8 +105,9 @@ jobs:
             arch: amd64
           - runner: ubuntu-26.04-arm
             arch: arm64
-          - runner: ubuntu-26.04-arm
-            arch: armv7
+            #  FIXME: https://github.com/searxng/searxng/pull/6655#issuecomment-5550293085
+            # - runner: ubuntu-26.04-arm
+            #   arch: armv7
 
     steps:
       - name: Login to GHCR
@@ -117,7 +118,7 @@ jobs:
           password: "${{ secrets.GITHUB_TOKEN }}"
 
       - name: Setup QEMU
-        uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8  # v4.2.0
+        uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a  # v4.3.0
 
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

.github/workflows/data-update.yml

@@ -31,7 +31,7 @@ jobs:
           - update_external_bangs.py
           - update_firefox_version.py
           - update_engine_traits.py
-          - update_wikidata_units.py
+          - update_wikidata.py
           - update_engine_descriptions.py
 
     permissions:

client/simple/generated/pygments.less

@@ -2,7 +2,7 @@
 
 /*
    this file is generated automatically by searxng_extra/update/update_pygments.py
-   using pygments version 2.20.0:
+   using pygments version 2.21.0:
 
        ./manage templates.simple.pygments
 */
@@ -114,14 +114,14 @@
     .gd { color: #FF4689 } /* Generic.Deleted */
     .ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
     .ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
-    .gr { color: #F8F8F2 } /* Generic.Error */
+    .gr { color: #FF4689 } /* Generic.Error */
     .gh { color: #F8F8F2 } /* Generic.Heading */
     .gi { color: #A6E22E } /* Generic.Inserted */
     .go { color: #66D9EF } /* Generic.Output */
     .gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
     .gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
     .gu { color: #959077 } /* Generic.Subheading */
-    .gt { color: #F8F8F2 } /* Generic.Traceback */
+    .gt { color: #66D9EF } /* Generic.Traceback */
     .kc { color: #66D9EF } /* Keyword.Constant */
     .kd { color: #66D9EF } /* Keyword.Declaration */
     .kn { color: #FF4689 } /* Keyword.Namespace */
@@ -132,7 +132,7 @@
     .m { color: #AE81FF } /* Literal.Number */
     .s { color: #E6DB74 } /* Literal.String */
     .na { color: #A6E22E } /* Name.Attribute */
-    .nb { color: #F8F8F2 } /* Name.Builtin */
+    .nb { color: #A6E22E } /* Name.Builtin */
     .nc { color: #A6E22E } /* Name.Class */
     .no { color: #66D9EF } /* Name.Constant */
     .nd { color: #A6E22E } /* Name.Decorator */
@@ -166,7 +166,7 @@
     .sr { color: #E6DB74 } /* Literal.String.Regex */
     .s1 { color: #E6DB74 } /* Literal.String.Single */
     .ss { color: #E6DB74 } /* Literal.String.Symbol */
-    .bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
+    .bp { color: #A6E22E } /* Name.Builtin.Pseudo */
     .fm { color: #A6E22E } /* Name.Function.Magic */
     .vc { color: #F8F8F2 } /* Name.Variable.Class */
     .vg { color: #F8F8F2 } /* Name.Variable.Global */

client/simple/package.json

@@ -23,27 +23,27 @@
     "not dead"
   ],
   "dependencies": {
-    "ionicons": "^8.0.13",
+    "ionicons": "^8.1.0",
     "normalize.css": "8.0.1",
-    "ol": "^10.9.0",
+    "ol": "^10.10.0",
     "swiped-events": "1.2.0"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.5.5",
-    "@types/node": "^26.1.1",
-    "browserslist": "^4.28.7",
+    "@biomejs/biome": "2.5.12",
+    "@types/node": "^26.5.0",
+    "browserslist": "^4.28.8",
     "browserslist-to-esbuild": "^2.1.1",
     "edge.js": "^6.5.1",
-    "less": "^4.8.0",
+    "less": "^4.9.0",
     "mathjs": "^15.2.0",
-    "sharp": "~0.35.3",
+    "sharp": "~0.35.4",
     "sort-package-json": "^4.0.0",
     "stylelint": "^17.14.1",
     "stylelint-config-standard-less": "^4.1.0",
     "stylelint-prettier": "^5.0.3",
-    "svgo": "^4.0.2",
+    "svgo": "^4.1.0",
     "typescript": "~7.0.2",
-    "vite": "^8.1.5",
+    "vite": "^8.2.2",
     "vite-bundle-analyzer": "^1.3.9"
   }
 }

client/simple/src/js/plugin/Calculator.ts

@@ -80,7 +80,12 @@ export default class Calculator extends Plugin {
 
     try {
       const node = Calculator.math.parse(searchInput.value);
-      return `${node.toString()} = ${node.evaluate()}`;
+      const value = node.evaluate();
+      if (typeof value !== "number") {
+        return;
+      }
+
+      return `${node.toString()} = ${value}`;
     } catch {
       // not a compatible math expression
       return;

client/simple/src/js/util/appendAnswerElement.ts

@@ -23,7 +23,7 @@ export const appendAnswerElement = (element: HTMLElement | string | number): voi
 
   if (!(element instanceof HTMLElement)) {
     const span = document.createElement("span");
-    span.innerHTML = element.toString();
+    span.textContent = element.toString();
     // biome-ignore lint/style/noParameterAssign: TODO
     element = span;
   }

docs/admin/settings/settings_engines.rst

@@ -58,10 +58,9 @@ engine is shown.  Most of the options have a default value or even are optional.
 
      # overwrite values from section 'outgoing:'
      enable_http2: false
+     enable_http3: false
      retries: 1
      max_connections: 100
-     max_keepalive_connections: 10
-     keepalive_expiry: 5.0
      using_tor_proxy: false
      proxies:
        http:
@@ -163,6 +162,16 @@ engine is shown.  Most of the options have a default value or even are optional.
 ``enable_http`` : optional
   Enable HTTP for this engine (by default only HTTPS is enabled).
 
+``enable_http3`` : optional
+  Use HTTP/3 (falls back to HTTP/2).  Default ``false``.
+  Ignored when a proxy is set.
+
+  .. hint::
+
+     HTTP/3 places demands on the IP infrastructure that are not met in every
+     environment.  Enable this option only if you are aware of these requirements
+     and the extent to which they are met.
+
 ``retry_on_http_error`` : optional
   Retry request on some HTTP status code.
 
@@ -179,20 +188,12 @@ engine is shown.  Most of the options have a default value or even are optional.
   Using tor proxy (``true``) or not (``false``) for this engine.  The default is
   taken from ``using_tor_proxy`` of the :ref:`settings outgoing`.
 
-.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
-
-``max_keepalive_connection#s`` :
-  `Pool limit configuration`_, overwrites value ``pool_maxsize`` from
-   :ref:`settings outgoing` for this engine.
+.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
 
 ``max_connections`` :
   `Pool limit configuration`_, overwrites value ``pool_connections`` from
   :ref:`settings outgoing` for this engine.
 
-``keepalive_expiry`` :
-  `Pool limit configuration`_, overwrites value ``keepalive_expiry`` from
-  :ref:`settings outgoing` for this engine.
-
 
 .. _private engines:
 

docs/admin/settings/settings_outgoing.rst

@@ -12,20 +12,12 @@ Communication with search engines.
      request_timeout: 2.0       # default timeout in seconds, can be override by engine
      max_request_timeout: 10.0  # the maximum timeout in seconds
      useragent_suffix: ""       # information like an email address to the administrator
-     pool_connections: 100      # Maximum number of allowable connections, or null
-                                # for no limits. The default is 100.
-     pool_maxsize: 10           # Number of allowable keep-alive connections, or null
-                                # to always allow. The default is 10.
-     enable_http2: true         # See https://www.python-httpx.org/http2/
+     pool_connections: 100      # Maximum number of concurrent connections (default: 100)
+     enable_http2: true         # Enables the use of HTTP2
      # uncomment below section if you want to use a custom server certificate
-     # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults
-     # and https://www.python-httpx.org/compatibility/#ssl-configuration
      #  verify: ~/.mitmproxy/mitmproxy-ca-cert.cer
      #
-     # uncomment below section if you want to use a proxyq see: SOCKS proxies
-     #   https://2.python-requests.org/en/latest/user/advanced/#proxies
-     # are also supported: see
-     #   https://2.python-requests.org/en/latest/user/advanced/#socks
+     # uncomment below section if you want to use a proxy
      #
      #  proxies:
      #    all://:
@@ -46,30 +38,26 @@ Communication with search engines.
   timeout to load).  Can be override by ``timeout`` in the :ref:`settings engines`.
 
 ``useragent_suffix`` :
-  Suffix to the user-agent SearXNG uses to send requests to others engines.  If an
-  engine wish to block you, a contact info here may be useful to avoid that.
+  Suffix to add when an engine's User-Agent is set via searxng_useragent().
+  Contact info here may be useful to avoid an engine blocking you.
 
-.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
-
-``pool_maxsize``:
-  Number of allowable keep-alive connections, or ``null`` to always allow.  The
-  default is 10.  See ``max_keepalive_connections`` `Pool limit configuration`_.
+.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
 
 ``pool_connections`` :
-  Maximum number of allowable connections, or ``null`` # for no limits.  The
-  default is 100.  See ``max_connections`` `Pool limit configuration`_.
+  Maximum number of concurrent connections.  The default is 100.
+  See ``max_clients`` `Pool limit configuration`_.
 
-``keepalive_expiry`` :
-  Number of seconds to keep a connection in the pool.  By default 5.0 seconds.
-  See ``keepalive_expiry`` `Pool limit configuration`_.
-
-.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying
+.. _curl_cffi proxies: https://curl-cffi.readthedocs.io/en/latest/quick_start.html
 
 ``proxies`` :
-  Define one or more proxies you wish to use, see `httpx proxies`_.
+  Define one or more proxies you wish to use, see `curl_cffi proxies`_.
   If there are more than one proxy for one protocol (http, https),
   requests to the engines are distributed in a round-robin fashion.
 
+  HTTP, HTTPS, SOCKS4, SOCKS5 and SOCKS5h proxies are supported
+  (``http://``, ``https://``, ``socks4://``, ``socks5://``, ``socks5h://``). You should
+  use ``socks5h://`` when using Tor so hostnames are resolved by the proxy.
+
 ``source_ips`` :
   If you use multiple network interfaces, define from which IP the requests must
   be made. Example:
@@ -87,18 +75,15 @@ Communication with search engines.
   different proxy and source ip.
 
 ``enable_http2`` :
-  Enable by default. Set to ``false`` to disable HTTP/2.
-
-.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults
-.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration
+  Enable by default (HTTP/2).  Set to ``false`` to force HTTP/1.1.
+  HTTP/3 is opt-in per engine (``enable_http3``).
 
 ``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR``
-  Allow to specify a path to certificate.
-  see `httpx verification defaults`_.
+  HTTPS verification uses the OS's trust store by default.
+  Set a path to use a custom CA file.
 
   In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and
   ``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.
-  see `httpx ssl configuration`_.
 
 ``max_redirects`` :
   30 by default. Maximum redirect before it is an error.

docs/dev/engines/engine_overview.rst

@@ -143,7 +143,7 @@ parameters with default value can be redefined for special purposes.
    data                   dict           ``{}``
    cookies                dict           ``{}``
    verify                 bool           ``True``
-   headers.User-Agent     str            a random User-Agent
+   headers.User-Agent     str            ``''``
    category               str            current category, like ``'general'``
    safesearch             int            ``0``, between ``0`` and ``2`` (normal, moderate, strict)
    time_range             Optional[str]  ``None``, can be ``day``, ``week``, ``month``, ``year``
@@ -229,6 +229,8 @@ following parameters can be used to specify a search request:
    max_redirects       int         maximum redirects, hard limit
    soft_max_redirects  int         maximum redirects, soft limit. Record an error but don't stop the engine
    raise_for_httperror bool        True by default: raise an exception if the HTTP code of response is >= 300
+   impersonate         str         curl_cffi impersonate target (default: chrome, none to disable)
+   curl_options        dict        Any extra libcurl options for the request
    =================== =========== ==========================================================================
 
 

docs/dev/engines/online/cara.rst

@@ -1,8 +0,0 @@
-.. _cara engine:
-
-===========
-Cara Images
-===========
-
-.. automodule:: searx.engines.cara
-   :members:

docs/dev/engines/online/europepmc.rst

@@ -0,0 +1,8 @@
+.. _europepmc engine:
+
+==========
+Europe PMC
+==========
+
+.. automodule:: searx.engines.europepmc
+   :members:

docs/dev/quickstart.rst

@@ -80,8 +80,8 @@ same environment, here are a few examples::
     # to test one of the update scripts
     (dev.env)$ searxng_extra/update/update_engine_traits.py --help
 
-    # to test the update of the wikidata units
-    (dev.env)$ searxng_extra/update/update_wikidata_units.py
+    # to test the update of the wikidata units and property names
+    (dev.env)$ searxng_extra/update/update_wikidata.py
 
 
 .. sidebar:: further read

docs/dev/reST.rst

@@ -286,7 +286,7 @@ content becomes smart.
    files & folders origin     :origin:`docs/dev/reST.rst`        ``:origin:`docs/dev/reST.rst```
    pull request               :pull:`4`                          ``:pull:`4```
    patch                      :patch:`af2cae6`                   ``:patch:`af2cae6```
-   PyPi package               :pypi:`httpx`                      ``:pypi:`httpx```
+   PyPi package               :pypi:`curl_cffi`                  ``:pypi:`curl_cffi```
    manual page man            :man:`bash`                        ``:man:`bash```
    intersphinx_
    --------------------------------------------------------------------------------------------------

docs/dev/searxng_extra/update.rst

@@ -90,10 +90,10 @@ Scripts to update static data in :origin:`searx/data/`
   :members:
 
 
-``update_wikidata_units.py``
+``update_wikidata.py``
 ============================
 
-:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
+:origin:`[source] <searxng_extra/update/update_wikidata.py>`
 
-.. automodule:: searxng_extra.update.update_wikidata_units
+.. automodule:: searxng_extra.update.update_wikidata
   :members:

requirements-dev.txt

@@ -2,9 +2,9 @@ mock==5.2.0
 nose2[coverage_plugin]==0.16.0
 cov-core==1.15.0
 black==25.9.0
-pylint==4.0.6
+pylint==4.0.8
 splinter==0.21.0
-selenium==4.46.0
+selenium==4.48.0
 Sphinx==8.2.3;python_version <= "3.11"
 Sphinx==9.1.0; python_version > "3.11"
 sphinx-issues==6.0.0
@@ -18,11 +18,11 @@ myst-parser==5.0.0
 linuxdoc==20260504
 aiounittest==1.5.0
 yamllint==1.38.0
-wlc==2.1.0
+wlc==2.1.1
 coloredlogs==15.0.1
 docutils>=0.21.2;python_version <= "3.11"
 docutils>=0.22.4; python_version > "3.11"
 parameterized==0.9.0
-granian[reload]==2.7.9
-basedpyright==1.39.9
+granian[reload]==2.8.2
+basedpyright==1.39.10
 types-lxml==2026.2.16

requirements-server.txt

@@ -1,2 +1,2 @@
-granian==2.7.9
-granian[pname]==2.7.9
+granian==2.8.2
+granian[pname]==2.8.2

requirements.txt

@@ -3,17 +3,15 @@ babel==2.18.0
 flask-babel==4.0.0
 flask==3.1.3
 jinja2==3.1.6
-lxml==6.1.1
-pygments==2.20.0
+lxml==6.1.2
+pygments==2.21.0
 python-dateutil==2.9.0.post0
 pyyaml==6.0.3
-httpx[http2]==0.28.1
-httpx-socks[asyncio]==0.10.0
-sniffio==1.3.1
+curl_cffi==0.16.1
 valkey==6.1.1
 markdown-it-py==4.2.0
 msgspec==0.21.1
-typer==0.27.0
+typer==0.27.2
 isodate==0.7.2
 whitenoise==6.12.0
 typing-extensions==4.16.0

searx/autocomplete.py

@@ -11,7 +11,7 @@ from urllib.parse import urlencode
 
 import lxml.etree
 import lxml.html
-from httpx import HTTPError
+from curl_cffi.requests.exceptions import RequestException
 
 from searx import settings
 from searx.engines import (
@@ -63,7 +63,7 @@ def bing(query: str, _sxng_locale: str) -> list[str]:
     base_url = "https://www.bing.com/AS/Suggestions?"
     # cvid has to be a 32 character long string consisting of numbers and uppsercase characters
     cvid = ''.join(random.choices(string.ascii_uppercase + string.digits, k=32))
-    response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}))
+    response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}), enable_http3=True)
     results: list[str] = []
 
     if response.ok:
@@ -83,7 +83,7 @@ def brave(query: str, _sxng_locale: str) -> list[str]:
     url = 'https://search.brave.com/api/suggest?'
     url += urlencode({'q': query})
     country = 'all'
-    kwargs = {'cookies': {'country': country}}
+    kwargs = {'cookies': {'country': country}, 'enable_http3': True}
     resp = get(url, **kwargs)
     results: list[str] = []
 
@@ -127,18 +127,17 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
 
 
 def google_complete(query: str, sxng_locale: str) -> list[str]:
-    """Autocomplete from Google.  Supports Google's languages and subdomains
+    """Autocomplete from Google.  Supports Google's languages
     (:py:obj:`searx.engines.google.get_google_info`) by using the async REST
     API::
 
-        https://{subdomain}/complete/search?{args}
+        https://www.google.com/complete/search?{args}
 
     """
 
     data = ENGINE_TRAITS.get("google") or {}
     traits = EngineTraits(**data)
     google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
-    url = 'https://{subdomain}/complete/search?{args}'
     args = urlencode(
         {
             'q': query,
@@ -148,7 +147,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
     )
     results: list[str] = []
 
-    resp = get(url.format(subdomain=google_info['subdomain'], args=args))
+    resp = get('https://www.google.com/complete/search?' + args, enable_http3=True)
     if resp and resp.ok:
         json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
         data = json.loads(json_txt)
@@ -419,5 +418,5 @@ def search_autocomplete(backend_name: str, query: str, sxng_locale: str) -> list
         return []
     try:
         return backend(query, sxng_locale)
-    except (HTTPError, SearxEngineResponseException):
+    except (RequestException, SearxEngineResponseException):
         return []

searx/botdetection/trusted_proxies.py

@@ -63,6 +63,20 @@ class ProxyFix:
         proxy_list: list[str] = cfg.get("botdetection.trusted_proxies", default=[])
         return [ip_network(net, strict=False) for net in proxy_list]
 
+    def is_trusted_proxy(
+        self,
+        addr: IPv4Address | IPv6Address | None,
+        trusted_proxies: list[IPv4Network | IPv6Network],
+    ) -> bool:
+        if addr is None:
+            return False
+
+        for net in trusted_proxies:
+            if addr.version == net.version and addr in net:
+                return True
+
+        return False
+
     def trusted_remote_addr(
         self,
         x_forwarded_for: list[IPv4Address | IPv6Address],
@@ -70,16 +84,8 @@ class ProxyFix:
     ) -> str:
         # always rtl
         for addr in reversed(x_forwarded_for):
-            trust: bool = False
-
-            for net in trusted_proxies:
-                if addr.version == net.version and addr in net:
-                    logger.debug("trust proxy %s (member of %s)", addr, net)
-                    trust = True
-                    break
-
-            # client address
-            if not trust:
+            if not self.is_trusted_proxy(addr, trusted_proxies):
+                logger.debug("client address from X-Forwarded-For: %s", addr)
                 return addr.compressed
 
         # fallback to first address
@@ -95,19 +101,21 @@ class ProxyFix:
         # in this function!
 
         orig_remote_addr: str | None = environ.pop("REMOTE_ADDR")
+        orig_remote_ip: IPv4Address | IPv6Address | None = None
 
         # Validate the IPs involved in this game and delete all invalid ones
         # from the WSGI environment.
 
         if orig_remote_addr:
             try:
-                addr = ip_address(orig_remote_addr)
-                if addr.version == 6 and addr.ipv4_mapped:
-                    addr = addr.ipv4_mapped
-                orig_remote_addr = addr.compressed
+                orig_remote_ip = ip_address(orig_remote_addr)
+                if orig_remote_ip.version == 6 and orig_remote_ip.ipv4_mapped:
+                    orig_remote_ip = orig_remote_ip.ipv4_mapped
+                orig_remote_addr = orig_remote_ip.compressed
             except ValueError as exc:
                 logger.error("REMOTE_ADDR: %s / discard REMOTE_ADDR from WSGI environment", exc)
                 orig_remote_addr = None
+                orig_remote_ip = None
 
         x_real_ip: str | None = environ.get("HTTP_X_REAL_IP")
         if x_real_ip:
@@ -141,11 +149,13 @@ class ProxyFix:
         if not x_forwarded_for and not x_real_ip:
             log_error_only_once("X-Forwarded-For nor X-Real-IP header is set!")
 
-        if x_forwarded_for and not trusted_proxies:
-            log_error_only_once("missing botdetection.trusted_proxies config")
-            # without trusted_proxies, this variable is useless for determining
-            # the real IP
-            x_forwarded_for = []
+        if x_forwarded_for or x_real_ip:
+            if not trusted_proxies:
+                log_error_only_once("missing botdetection.trusted_proxies config")
+
+            if not self.is_trusted_proxy(orig_remote_ip, trusted_proxies):
+                x_forwarded_for = []
+                x_real_ip = None
 
         # securing the WSGI environment variables that are adjusted
 

searx/data/__init__.py

@@ -32,6 +32,13 @@ class WikiDataUnitType(t.TypedDict):
     to_si_factor: float
 
 
+WikiDataPropertyNameType = str | dict[str, str]
+"""Name of a Wikidata property. Can be either the plain name or a dictionary of
+language code to property name, e.g. ``{"en": "Date of birth"}``."""
+WikiDataPropertiesType = dict[str, WikiDataPropertyNameType]
+"""Dictionary from wikidata property ID to property name."""
+
+
 class LocalesType(t.TypedDict):
     """Data structure of an item in ``locales.json``"""
 
@@ -41,6 +48,7 @@ class LocalesType(t.TypedDict):
 
 USER_AGENTS: UserAgentType
 WIKIDATA_UNITS: dict[str, WikiDataUnitType]
+WIKIDATA_PROPERTIES: WikiDataPropertiesType
 TRACKER_PATTERNS: TrackerPatternsDB
 LOCALES: LocalesType
 CURRENCIES: CurrenciesDB
@@ -52,11 +60,12 @@ ENGINE_DESCRIPTIONS: dict[str, dict[str, t.Any]]
 ENGINE_TRAITS: dict[str, dict[str, t.Any]]
 
 
-lazy_globals = {
+lazy_globals: dict[str, t.Any] = {
     "CURRENCIES": CurrenciesDB(),
     "USER_AGENTS": None,
     "EXTERNAL_URLS": None,
     "WIKIDATA_UNITS": None,
+    "WIKIDATA_PROPERTIES": None,
     "EXTERNAL_BANGS": None,
     "OSM_KEYS_TAGS": None,
     "ENGINE_DESCRIPTIONS": None,
@@ -69,6 +78,7 @@ data_json_files = {
     "USER_AGENTS": "useragents.json",
     "EXTERNAL_URLS": "external_urls.json",
     "WIKIDATA_UNITS": "wikidata_units.json",
+    "WIKIDATA_PROPERTIES": "wikidata_properties.json",
     "EXTERNAL_BANGS": "external_bangs.json",
     "OSM_KEYS_TAGS": "osm_keys_tags.json",
     "ENGINE_DESCRIPTIONS": "engine_descriptions.json",

searx/data/tracker_patterns.py

@@ -10,7 +10,7 @@ import re
 from collections.abc import Iterator
 from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
 
-from httpx import HTTPError
+from curl_cffi.requests.exceptions import RequestException
 
 from searx.data.core import get_cache, log
 from searx.network import get as http_get
@@ -28,11 +28,11 @@ class TrackerPatternsDB:
 
     ctx_name = "data_tracker_patterns"
 
+    # ClearURL rule lists, the first one that responds HTTP 200 is used
     CLEAR_LIST_URL = [
-        # ClearURL rule lists, the first one that responds HTTP 200 is used
-        "https://rules1.clearurls.xyz/data.minify.json",
+        "https://cdn.jsdelivr.net/gh/clearurls/rules@refs/heads/gh-pages/data.minify.json",
         "https://rules2.clearurls.xyz/data.minify.json",
-        "https://raw.githubusercontent.com/ClearURLs/Rules/refs/heads/master/data.min.json",
+        "https://rules1.clearurls.xyz/data.minify.json",
     ]
 
     class Fields:
@@ -87,8 +87,8 @@ class TrackerPatternsDB:
             try:
                 resp = http_get(url, timeout=3)
 
-            except HTTPError as exc:
-                log.warning("TRACKER_PATTERNS: HTTPError (%s) occured while fetching %s", url, exc)
+            except RequestException as exc:
+                log.warning("TRACKER_PATTERNS: RequestException while fetching %s: %s", url, exc)
                 continue
 
             if resp.status_code != 200:

searx/data/useragents.json

@@ -5,7 +5,7 @@
     ],
     "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
     "versions": [
-        "153.0",
-        "152.0"
+        "154.0",
+        "153.0"
     ]
 }

searx/data/wikidata_units.json

@@ -3474,11 +3474,6 @@
         "symbol": "mm⁻²",
         "to_si_factor": 1e-06
     },
-    "Q136039973": {
-        "si_name": "Q6137407",
-        "symbol": "FPS",
-        "to_si_factor": 1.0
-    },
     "Q1361854": {
         "si_name": "Q11570",
         "symbol": "dwt",
@@ -5254,6 +5249,11 @@
         "symbol": "μA",
         "to_si_factor": 1e-06
     },
+    "Q31274648": {
+        "si_name": "Q6137407",
+        "symbol": "FPS",
+        "to_si_factor": 1.0
+    },
     "Q3186734": {
         "si_name": "Q3186734",
         "symbol": "J/(m³ K)",

searx/enginelib/__init__.py

@@ -305,7 +305,7 @@ class Engine(abc.ABC):  # pylint: disable=too-few-public-methods
 
     region: str = ""
     """For an engine, when there is ``region: ...`` in the YAML settings the engine
-    does support only this one region::
+    does support only this one region:
 
     .. code:: yaml
 
@@ -317,6 +317,9 @@ class Engine(abc.ABC):  # pylint: disable=too-few-public-methods
     enable_http: bool
     """Enable HTTP (by default only HTTPS is enabled)."""
 
+    enable_http3: bool = False
+    """Enables the use of HTTP/3 if available"""
+
     shortcut: str
     """Code used to execute bang requests (``!foo``)"""
 

searx/engines/360search_videos.py

@@ -6,7 +6,8 @@ from urllib.parse import urlencode
 from datetime import datetime
 
 from searx.exceptions import SearxEngineAPIException
-from searx.utils import html_to_text, get_embeded_stream_url
+from searx.result_types import EngineResults
+from searx.utils import html_to_text
 
 about = {
     "website": "https://tv.360kan.com/",
@@ -29,12 +30,12 @@ def request(query, params):
     return params
 
 
-def response(resp):
+def response(resp) -> EngineResults:
     try:
         data = resp.json()
     except Exception as e:
         raise SearxEngineAPIException(f"Invalid response: {e}") from e
-    results = []
+    res = EngineResults()
 
     if "data" not in data or "result" not in data["data"]:
         raise SearxEngineAPIException("Invalid response")
@@ -50,16 +51,15 @@ def response(resp):
             except (ValueError, TypeError):
                 published_date = None
 
-        results.append(
-            {
-                'url': entry["play_url"],
-                'title': html_to_text(entry["title"]),
-                'content': html_to_text(entry["description"]),
-                'template': 'videos.html',
-                'publishedDate': published_date,
-                'thumbnail': entry["cover_img"],
-                "iframe_src": get_embeded_stream_url(entry["play_url"]),
-            }
+        res.add(
+            res.types.LegacyResult(
+                url=entry["play_url"],
+                title=html_to_text(entry["title"]),
+                content=html_to_text(entry["description"]),
+                template='videos.html',
+                publishedDate=published_date,
+                thumbnail=entry["cover_img"],
+            )
         )
 
-    return results
+    return res

searx/engines/__builtins__.pyi

@@ -26,6 +26,7 @@ categories: list[str]
 disabled: bool
 display_error_messages: bool
 enable_http: bool
+enable_http3: bool
 engine_type: str
 inactive: bool
 max_page: int

searx/engines/__init__.py

@@ -187,8 +187,9 @@ def set_loggers(engine: "Engine|types.ModuleType", engine_name: str):
 def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]):
     # pylint: disable=too-many-branches
 
-    # set engine attributes from engine_data
+    # set / update engine attributes from engine_data
     kvargs: dict[str, t.Any]
+    engine.about = getattr(engine, "about", EngineAbout())
     if isinstance(engine.about, EngineAbout):
         kvargs = {**msgspec.to_builtins(engine.about), **engine_data.get("about", {})}
     else:

searx/engines/adobe_stock.py

@@ -35,7 +35,7 @@ Implementation
 ==============
 
 """
-
+import typing as t
 from datetime import datetime, timedelta
 from urllib.parse import urlencode
 
@@ -85,7 +85,7 @@ Additional subcategories:
 # Do we need support for "free_collection" and "include_stock_enterprise"?
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if not categories:
         raise ValueError("adobe_stock engine: categories is unset")
 
@@ -100,9 +100,9 @@ def init(_):
         raise ValueError("adobe_stock engine: adobe_content_types is unset")
 
     if isinstance(adobe_content_types, list):
-        for t in adobe_content_types:
-            if t not in ADOBE_VALID_TYPES:
-                raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % t)
+        for content_type in adobe_content_types:
+            if content_type not in ADOBE_VALID_TYPES:
+                raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % content_type)
     else:
         raise ValueError(
             "adobe_stock engine: adobe_content_types must be a list of strings not %s" % type(adobe_content_types)

searx/engines/baidu.py

@@ -49,6 +49,9 @@ CACHE: EngineCache
 
 
 def setup(engine_settings: dict[str, t.Any]) -> bool:
+    if baidu_category not in ('general', 'images', 'it'):
+        raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
+
     global CACHE  # pylint: disable=global-statement
     CACHE = EngineCache(engine_settings["name"])
     return True
@@ -65,11 +68,6 @@ def get_image_cookies(headers: dict[str, str]) -> dict[str, str]:
     return cookies
 
 
-def init(_):
-    if baidu_category not in ('general', 'images', 'it'):
-        raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
-
-
 def request(query, params):
     page_num = params["pageno"]
 

searx/engines/bing.py

@@ -40,6 +40,7 @@ about: dict[str, t.Any] = {
 # engine dependent config
 categories = ["general", "web"]
 safesearch = True
+enable_http3 = True
 _safesearch_map: dict[int, str] = {
     0: "off",
     1: "moderate",
@@ -71,43 +72,21 @@ def get_locale_params(engine_region: str | None) -> dict[str, str] | None:
     return {"mkt": engine_region}
 
 
-def override_accept_language(params: "OnlineParams", engine_region: str | None) -> None:
-    """Override the ``Accept-Language`` header.
-
-    The default header built by :py:class:`~searx.search.processors.online.OnlineProcessor`
-    appends ``en;q=0.3`` as a fallback language::
-
-        Accept-Language: de,de-DE;q=0.7,en;q=0.3
-
-    Bing seems to better select the results locale based on the
-    ``Accept-Language`` value header.
-
-    This function is shared with :py:mod:`searx.engines.bing_images`,
-    :py:mod:`searx.engines.bing_news`, and :py:mod:`searx.engines.bing_videos`.
-    """
-
-    if not engine_region or engine_region == "clear":
-        return
-
-    lang = engine_region.split("-")[0]
-    params["headers"]["Accept-Language"] = f"{engine_region},{lang};q=0.9"
-
-
 def request(query: str, params: "OnlineParams"):
     """Assemble a Bing-Web request."""
 
     engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
 
-    override_accept_language(params, engine_region)
-
     query_params: dict[str, str | int] = {
         "q": query,
         "adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
     }
 
-    locale_params = get_locale_params(engine_region)
-    if locale_params:
-        query_params.update(locale_params)
+    if engine_region and engine_region != "clear":
+        lang, _, cc = engine_region.partition("-")
+        query_params["setlang"] = lang
+        if cc and cc not in ("us", "cn", "ru"):  # bing just sends junk for these
+            query_params["cc"] = cc
 
     params["url"] = f"{base_url}/search?{urlencode(query_params)}"
 

searx/engines/bing_images.py

@@ -9,7 +9,6 @@ from lxml import html
 from searx.engines.bing import (  # pylint: disable=unused-import
     fetch_traits,
     get_locale_params,
-    override_accept_language,
 )
 
 # about
@@ -25,6 +24,7 @@ about = {
 # engine dependent config
 categories = ["images", "web"]
 paging = True
+enable_http3 = True
 safesearch = True
 time_range_support = True
 time_map = {
@@ -43,8 +43,6 @@ def request(query, params):
 
     engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
 
-    override_accept_language(params, engine_region)
-
     # build URL query
     # - example: https://www.bing.com/images/async?q=foo&async=1&first=1&count=35
     query_params = {

searx/engines/bing_news.py

@@ -12,10 +12,7 @@ from urllib.parse import urlencode
 from lxml import html
 
 from searx.enginelib.traits import EngineTraits
-from searx.engines.bing import (
-    get_locale_params,
-    override_accept_language,
-)
+from searx.engines.bing import get_locale_params
 from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
 
 # about
@@ -33,6 +30,7 @@ categories = ["news"]
 paging = True
 """If go through the pages and there are actually no new results for another
 page, then bing returns the results from the last page again."""
+enable_http3 = True
 
 time_range_support = True
 time_map = {
@@ -53,8 +51,6 @@ def request(query, params):
 
     engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
 
-    override_accept_language(params, engine_region)
-
     # build URL query
     # - example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
     page = int(params.get("pageno", 1)) - 1

searx/engines/bing_videos.py

@@ -9,7 +9,6 @@ from lxml import html
 from searx.engines.bing import (  # pylint: disable=unused-import
     fetch_traits,
     get_locale_params,
-    override_accept_language,
 )
 from searx.engines.bing_images import time_map
 from searx.utils import eval_xpath, eval_xpath_getindex
@@ -26,6 +25,7 @@ about = {
 # engine dependent config
 categories = ["videos", "web"]
 paging = True
+enable_http3 = True
 safesearch = True
 time_range_support = True
 
@@ -38,8 +38,6 @@ def request(query, params):
 
     engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
 
-    override_accept_language(params, engine_region)
-
     # build URL query
     # - example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
     query_params = {

searx/engines/boardreader.py

@@ -45,7 +45,7 @@ CACHE_SESSION_ID_KEY = "session_id_key"
 KEYWORD_RE = re.compile(r"\[\/?Keyword\]")
 
 
-def init(engine_settings: dict[str, t.Any]) -> bool:
+def setup(engine_settings: dict[str, t.Any]) -> bool:
     global CACHE  # pylint: disable=global-statement
     CACHE = EngineCache(engine_name=engine_settings["name"])
     return True

searx/engines/brave.py

@@ -135,7 +135,7 @@ from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
     extract_text,
-    get_embeded_stream_url,
+    get_embedded_stream_url,
     js_obj_str_to_json_str,
     js_obj_str_to_python,
 )
@@ -151,6 +151,7 @@ about = {
 
 base_url = "https://search.brave.com/"
 categories = []
+enable_http3 = True
 brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = "search"
 """Brave supports common web-search, videos, images, news, and goggles search.
 
@@ -247,13 +248,13 @@ def extract_json_data(text: str) -> dict[str, t.Any]:
     #    node_ids: [0, 19],
     #    data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]
     #          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-    text = text[text.index("<script") : text.index("</script")]
-    if not text:
-        raise ValueError("can't find JS/JSON data in the given text")
+    #    form: null,
+    #    error: null
+    # });
     start = text.index("data: [{")
-    end = text.rindex("}}]")
-    js_obj_str = text[start:end]
-    js_obj_str = "{" + js_obj_str + "}}]}"
+    newline = text.index("\n", start)
+    end = text.rindex("}}]", start, newline)
+    js_obj_str = "{" + text[start:end] + "}}]}"
     # js_obj_str = js_obj_str.replace("\xa0", "")  # remove ASCII for &nbsp;
     # js_obj_str = js_obj_str.replace(r"\u003C", "<").replace(r"\u003c", "<")  # fix broken HTML tags in strings
     json_str = js_obj_str_to_json_str(js_obj_str)
@@ -338,7 +339,7 @@ def _parse_search(resp: SXNG_Response) -> EngineResults:
         if len(video_tag):
             # In my tests a video tag in the WEB search was most often not a
             # video, except the ones from youtube ..
-            iframe_src = get_embeded_stream_url(url)
+            iframe_src = get_embedded_stream_url(url)
             if iframe_src:
                 item["iframe_src"] = iframe_src
                 item["template"] = "videos.html"
@@ -353,14 +354,14 @@ def _parse_news(resp: SXNG_Response) -> EngineResults:
     res = EngineResults()
     dom = html.fromstring(resp.text)
 
-    for result in eval_xpath_list(dom, "//div[contains(@class, 'results')]//div[@data-type='news']"):
-        url = eval_xpath_getindex(result, ".//a[contains(@class, 'result-header')]/@href", 0, default=None)
+    for result in eval_xpath_list(dom, "//div[@data-type='news']"):
+        url = eval_xpath_getindex(result, ".//a/@href", 0, default=None)
         if url is None:
             continue
 
-        title = eval_xpath_list(result, ".//span[contains(@class, 'snippet-title')]")
-        content = eval_xpath_list(result, ".//p[contains(@class, 'desc')]")
-        thumbnail = eval_xpath_getindex(result, ".//div[contains(@class, 'image-wrapper')]//img/@src", 0, default="")
+        title = eval_xpath_list(result, ".//div[contains(@class, 'title')]")
+        content = eval_xpath_list(result, ".//div[contains(@class, 'description')]")
+        thumbnail = eval_xpath_getindex(result, ".//a[contains(@class, 'thumbnail')]//img/@src", 0, default="")
 
         item = res.types.LegacyResult(
             template="default.html",
@@ -406,9 +407,6 @@ def _parse_videos(json_resp: dict[str, t.Any]) -> EngineResults:
         )
         if result["thumbnail"] is not None:
             item["thumbnail"] = result["thumbnail"]["src"]
-        iframe_src = get_embeded_stream_url(result["url"])
-        if iframe_src:
-            item["iframe_src"] = iframe_src
 
         res.add(item)
 

searx/engines/braveapi.py

@@ -40,7 +40,7 @@ if t.TYPE_CHECKING:
 about = {
     "website": "https://api.search.brave.com/",
     "wikidata_id": None,
-    "official_api_documentation": "https://api-dashboard.search.brave.com/documentation",
+    "official_api_documentation": "https://api-dashboard.search.brave.com/api-reference/web/search/get",
     "use_official_api": True,
     "require_api_key": True,
     "results": "JSON",
@@ -63,8 +63,10 @@ base_url = "https://api.search.brave.com/res/v1/web/search"
 time_range_map = {"day": "past_day", "week": "past_week", "month": "past_month", "year": "past_year"}
 """Mapping of SearXNG time ranges to Brave API time ranges."""
 
+max_page = 10
 
-def init(_):
+
+def setup(_: dict[str, t.Any]) -> bool | None:
     """Initialize the engine."""
     if not api_key:
         raise SearxEngineAPIException("No API key provided")
@@ -75,7 +77,7 @@ def request(query: str, params: "OnlineParams") -> None:
     search_args: dict[str, str | int | None] = {
         "q": query,
         "count": results_per_page,
-        "offset": (params["pageno"] - 1) * results_per_page,
+        "offset": params["pageno"] - 1,
         "text_decorations": False,
     }
 
@@ -89,6 +91,7 @@ def request(query: str, params: "OnlineParams") -> None:
 
     params["url"] = f"{base_url}?{urlencode(search_args)}"
     params["headers"]["X-Subscription-Token"] = api_key
+    params["headers"]["Accept"] = "application/json"
 
 
 def _extract_published_date(published_date_raw: str):

searx/engines/cara.py

@@ -1,85 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-# pylint: disable=invalid-name
-"""Cara_ is a social media and portfolio-sharing platform for artists and art
-enthusiasts.
-
-With the widespread use of generative AI, Cara_ decided to build a place that
-filters out gen AI images so that people searching for authentic creatives and
-images can do so easily.
-
-.. _Cara: https://cara.app/about
-"""
-
-from urllib.parse import urlencode
-
-import typing as t
-
-from searx.result_types import EngineResults
-
-if t.TYPE_CHECKING:
-    from searx.extended_types import SXNG_Response
-    from searx.search.processors import OnlineParams
-
-
-about = {
-    "website": "https://cara.app",
-    "official_api_documentation": None,
-    "use_official_api": False,
-    "require_api_key": False,
-    "results": "JSON",
-}
-
-base_url = "https://cara.app"
-images_url = "https://images.cara.app"
-
-categories = ["images"]
-paging = True
-results_per_page = 24
-
-# if using HTTP2, we get blocked immediately
-enable_http2 = False
-
-
-def request(query: str, params: "OnlineParams") -> None:
-    args = {
-        "q": query,
-        "sortBy": "Top",
-        "take": results_per_page,
-        "skip": (params["pageno"] - 1) * results_per_page,
-    }
-    params["url"] = f"{base_url}/api/search/portfolio-posts?{urlencode(args)}"
-
-
-def response(resp: "SXNG_Response"):
-    res = EngineResults()
-    json_data: list[dict[str, t.Any]] = resp.json()
-
-    for result in json_data:
-        thumbnail, img = None, None
-
-        i: dict[str, str]
-        for i in result["images"]:
-            if thumbnail is None or i["isCoverImg"]:
-                thumbnail = i
-
-            if img is None or not i["isCoverImg"]:
-                img = i
-
-        if not thumbnail or not img:
-            continue
-
-        res.add(
-            res.types.LegacyResult(
-                {
-                    "template": "images.html",
-                    "url": f"{base_url}/post/{result['id']}",
-                    "thumbnail_src": f"{images_url}/{thumbnail['src']}?height=256",
-                    "img_src": f"{images_url}/{img['src']}",
-                    "title": result["title"],
-                    "content": result["content"],
-                    "author": result["name"],
-                }
-            )
-        )
-
-    return res

searx/engines/chatnoir.py

@@ -41,7 +41,7 @@ search_index = "cw22"
 <https://www.chatnoir.eu/docs/api-general>`_ for a full list."""
 
 
-def _obtain_api_key() -> tuple[str, str, str]:
+def _obtain_api_key() -> tuple[str, str]:
     home_resp = get(base_url)
     if not home_resp.ok:
         raise SearxEngineAPIException("failed to obtain api key")
@@ -58,10 +58,9 @@ def _obtain_api_key() -> tuple[str, str, str]:
     )
     if not token_resp.ok:
         raise SearxEngineAPIException("failed to obtain api key")
-    session_id = token_resp.cookies["sessionid"]
     scraped_api_key = token_resp.json()["token"]["token"]
 
-    return csrf_token, session_id, scraped_api_key
+    return csrf_token, scraped_api_key
 
 
 def request(query: str, params: "OnlineParams"):
@@ -73,7 +72,7 @@ def request(query: str, params: "OnlineParams"):
 
         params["headers"].update(headers)
     else:
-        csrf_token, session_id, scraped_api_key = _obtain_api_key()
+        csrf_token, scraped_api_key = _obtain_api_key()
 
         headers = {
             "Authorization": f"Bearer {scraped_api_key}",
@@ -81,10 +80,11 @@ def request(query: str, params: "OnlineParams"):
         }
 
         params["headers"].update(headers)
-        params["cookies"] = {"csrftoken": session_id, "sessionid": session_id}
+        params["cookies"] = {"csrftoken": csrf_token}
 
     params["url"] = f"{base_url}/api/v1/_search"
     params["method"] = "POST"
+    params["impersonate"] = "none"
 
     json_data = {
         "query": query,

searx/engines/chinaso.py

@@ -78,7 +78,7 @@ time_range_dict = {'day': '24h', 'week': '1w', 'month': '1m', 'year': '1y'}
 base_url = "https://www.chinaso.com"
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if chinaso_news_source not in t.get_args(ChinasoNewsSourceType):
         raise ValueError(f"Unsupported news source: {chinaso_news_source}")
 

searx/engines/command.py

@@ -74,6 +74,7 @@ Implementations
 ===============
 """
 
+import typing as t
 import re
 from os.path import expanduser, isabs, realpath, commonprefix
 from shlex import split as shlex_split
@@ -100,7 +101,7 @@ _command_logger = logger.getChild('command')
 _compiled_parse_regex = {}
 
 
-def init(engine_settings):
+def setup(engine_settings: dict[str, t.Any]) -> bool | None:
     check_parsing_options(engine_settings)
 
     if 'command' not in engine_settings:

searx/engines/core.py

@@ -141,12 +141,13 @@ def response(resp: "SXNG_Response") -> EngineResults:
             if name:
                 authors.add(name)
 
+        tag = result.get("fieldOfStudy")
         res.add(
             res.types.Paper(
                 title=result.get("title"),
                 url=url,
                 content=result.get("fullText", "") or "",
-                tags=result.get("fieldOfStudy", []),
+                tags=[tag] if tag else [],
                 publishedDate=published_date,
                 type=result.get("documentType", "") or "",
                 authors=authors,

searx/engines/deviantart.py

@@ -1,11 +1,18 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Deviantart (Images)"""
 
+import typing as t
+
 import urllib.parse
 from lxml import html
 
+from searx.result_types import EngineResults
 from searx.utils import extract_text, eval_xpath, eval_xpath_list
 
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
+
 # about
 about = {
     "website": 'https://www.deviantart.com/',
@@ -23,63 +30,62 @@ paging = True
 # search-url
 base_url = 'https://www.deviantart.com'
 
-results_xpath = '//div[@class="V_S0t_"]/div/div/a'
-url_xpath = './@href'
-thumbnail_src_xpath = './div/img/@src'
-img_src_xpath = './div/img/@srcset'
-title_xpath = './@aria-label'
-premium_xpath = '../div/div/div/text()'
-premium_keytext = 'Watch the artist to view this deviation'
-cursor_xpath = '(//a[@class="vQ2brP"]/@href)[last()]'
+results_xpath = '//div[@data-testid="content_row"]//a[.//*[@data-testid="thumb"]]'
+img_src_xpath = './/img/@srcset'
+thumbnail_src_xpath = './/img/@src'
+author_xpath = './/*[@property="schema:name"]/@content'
+cursor_xpath = '//a[contains(@href, "cursor=") and contains(., "Next")]/@href'
 
 
-def request(query, params):
-
+def request(query: str, params: "OnlineParams"):
     # https://www.deviantart.com/search?q=foo
 
-    nextpage_url = params['engine_data'].get('nextpage')
-    # don't use nextpage when user selected to jump back to page 1
-    if params['pageno'] > 1 and nextpage_url is not None:
-        params['url'] = nextpage_url
-    else:
-        params['url'] = f"{base_url}/search?{urllib.parse.urlencode({'q': query})}"
+    args = {'q': query}
+    if params['pageno'] > 1:
+        cursor = params['engine_data'].get('cursor')
+        if cursor:
+            args['cursor'] = cursor
 
-    return params
+    params['url'] = f"{base_url}/search?{urllib.parse.urlencode(args)}"
 
 
-def response(resp):
+def response(resp: "SXNG_Response") -> EngineResults:
 
-    results = []
+    res = EngineResults()
     dom = html.fromstring(resp.text)
 
     for result in eval_xpath_list(dom, results_xpath):
-        # skip images that are blurred
-        _text = extract_text(eval_xpath(result, premium_xpath))
-        if _text and premium_keytext in _text:
-            continue
+        thumbnail_src = extract_text(eval_xpath(result, thumbnail_src_xpath))
         img_src = extract_text(eval_xpath(result, img_src_xpath))
+        # mature locked thumbs have blur transform (blur_15, blur_30 etc..)
+        if ',blur_' in f'{thumbnail_src}{img_src}':
+            continue
         if img_src:
             img_src = img_src.split(' ')[0]
             parsed_url = urllib.parse.urlparse(img_src)
             img_src = parsed_url._replace(path=parsed_url.path.split('/v1')[0]).geturl()
 
-        results.append(
-            {
-                'template': 'images.html',
-                'url': extract_text(eval_xpath(result, url_xpath)),
-                'img_src': img_src,
-                'thumbnail_src': extract_text(eval_xpath(result, thumbnail_src_xpath)),
-                'title': extract_text(eval_xpath(result, title_xpath)),
-            }
+        author = extract_text(eval_xpath(result, author_xpath))
+
+        res.add(
+            res.types.Image(
+                template='images.html',
+                url=result.get('href'),
+                img_src=img_src or "",
+                thumbnail_src=thumbnail_src or "",
+                title=result.get('aria-label'),
+                author=author or "",
+            )
         )
 
     nextpage_url = extract_text(eval_xpath(dom, cursor_xpath))
-    if nextpage_url:
-        results.append(
-            {
-                'engine_data': nextpage_url.replace("http://", "https://"),
-                'key': 'nextpage',
-            }
+    cursor = urllib.parse.parse_qs(urllib.parse.urlparse(nextpage_url or '').query).get('cursor', [None])[0]
+    if cursor:
+        res.add(
+            res.types.LegacyResult(
+                engine_data=cursor,
+                key='cursor',
+            )
         )
 
-    return results
+    return res

searx/engines/dogpile.py

@@ -8,6 +8,9 @@ import typing as t
 from datetime import datetime, timezone
 import html
 
+from searx.enginelib import EngineCache
+from searx.exceptions import SearxEngineAPIException
+from searx.network import post
 from searx.utils import format_duration, html_to_text, humanize_number
 from searx.result_types import EngineResults
 
@@ -35,15 +38,36 @@ dogpile_categ = "search"
 base_url = "https://www.dogpile.com"
 safe_search_map = {0: "none", 1: "moderate", 2: "heavy"}
 
+CACHE: EngineCache
+"""Cache for the API token from dogpile"""
 
-def init(_):
+
+def setup(_: dict[str, t.Any]) -> bool | None:
     if dogpile_categ not in ("search", "images", "videos", "news"):
         raise ValueError("invalid search type: %s" % dogpile_categ)
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache("dogpile")  # one token for images/videos/news
+    return True
+
+
+def _obtain_token() -> str:
+    token = CACHE.get("token")
+    if token:
+        return token
+    resp = post(f"{base_url}/api/token/refresh", headers={"Origin": base_url}, cookies={"dp_api_token": "1"})
+    if not resp.ok:
+        raise SearxEngineAPIException("failed to obtain dogpile token")
+    token = resp.json()["token"]
+    CACHE.set("token", token, expire=240)  # 300s ttl
+    return token
 
 
 def request(query: str, params: "OnlineParams"):
     params["url"] = f"{base_url}/api/{dogpile_categ}"
     params["headers"]["Origin"] = base_url
+    params["cookies"]["dp_api_token"] = "1"
+    if dogpile_categ != "search":  # web doesnt need token
+        params["headers"]["x-dogpile-token"] = _obtain_token()
 
     params["method"] = "POST"
     params["json"] = {"q": query, "qadf": safe_search_map[params["safesearch"]], "page": params["pageno"]}

searx/engines/duckduckgo_extra.py

@@ -10,7 +10,8 @@ from datetime import datetime
 from urllib.parse import urlencode
 from urllib.parse import quote_plus
 
-from searx.utils import get_embeded_stream_url, html_to_text, gen_useragent, extr
+from searx.result_types import EngineResults, MainResult, LegacyResult, Image
+from searx.utils import html_to_text, gen_useragent, extr
 from searx.network import get  # see https://github.com/searxng/searxng/issues/762
 
 from searx.engines.duckduckgo import fetch_traits  # pylint: disable=unused-import
@@ -47,7 +48,7 @@ _HTTP_User_Agent: str = gen_useragent()
 send_accept_language_header = False
 
 
-def init(engine_settings: dict[str, t.Any]):
+def setup(engine_settings: dict[str, t.Any]) -> bool | None:
 
     if engine_settings["ddg_category"] not in ["images", "videos", "news"]:
         raise ValueError(f"Unsupported DuckDuckGo category: {engine_settings['ddg_category']}")
@@ -97,6 +98,7 @@ def request(query: str, params: "OnlineParams") -> None:
     # The vqd value is generated from the query and the UA header. To be able to
     # reuse the vqd value, the UA header must be static.
     headers["User-Agent"] = _HTTP_User_Agent
+    params["impersonate"] = "none"
     vqd = get_vqd(query=query, params=params) or fetch_vqd(query=query, params=params)
 
     headers["Accept"] = "*/*"
@@ -148,54 +150,51 @@ def request(query: str, params: "OnlineParams") -> None:
 
 
 def _image_result(result):
-    return {
-        'template': 'images.html',
-        'url': result['url'],
-        'title': result['title'],
-        'content': '',
-        'thumbnail_src': result['thumbnail'],
-        'img_src': result['image'],
-        'resolution': '%s x %s' % (result['width'], result['height']),
-        'source': result['source'],
-    }
+    return Image(
+        url=result['url'],
+        title=result['title'],
+        content='',
+        thumbnail_src=result['thumbnail'],
+        img_src=result['image'],
+        resolution='%s x %s' % (result['width'], result['height']),
+        source=result['source'],
+    )
 
 
 def _video_result(result):
-    return {
-        'template': 'videos.html',
-        'url': result['content'],
-        'title': result['title'],
-        'content': result['description'],
-        'thumbnail': result['images'].get('small') or result['images'].get('medium'),
-        'iframe_src': get_embeded_stream_url(result['content']),
-        'source': result['provider'],
-        'length': result['duration'],
-        'metadata': result.get('uploader'),
-    }
+    return LegacyResult(
+        template='videos.html',
+        url=result['content'],
+        title=result['title'],
+        content=result['description'],
+        thumbnail=result['images'].get('small') or result['images'].get('medium'),
+        source=result['provider'],
+        length=result['duration'],
+        metadata=result.get('uploader'),
+    )
 
 
 def _news_result(result):
-    return {
-        'url': result['url'],
-        'title': result['title'],
-        'content': html_to_text(result['excerpt']),
-        'source': result['source'],
-        'publishedDate': datetime.fromtimestamp(result['date']),
-    }
+    return MainResult(
+        url=result['url'],
+        title=result['title'],
+        content=html_to_text(result['excerpt']),
+        publishedDate=datetime.fromtimestamp(result['date']),
+    )
 
 
-def response(resp):
-    results = []
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
     res_json = resp.json()
 
     for result in res_json['results']:
         if ddg_category == 'images':
-            results.append(_image_result(result))
+            res.add(_image_result(result))
         elif ddg_category == 'videos':
-            results.append(_video_result(result))
+            res.add(_video_result(result))
         elif ddg_category == 'news':
-            results.append(_news_result(result))
+            res.add(_news_result(result))
         else:
             raise ValueError(f"Invalid duckduckgo category: {ddg_category}")
 
-    return results
+    return res

searx/engines/duckduckgo_weather.py

@@ -109,7 +109,19 @@ def response(resp: SXNG_Response):
 
     json_data = loads(resp.text[resp.text.find('\n') + 1 : resp.text.rfind('\n') - 2])
 
-    geoloc = weather.GeoLocation.by_query(resp.search_params["query"])
+    location = json_data.get("location")
+    if not location:
+        return res
+
+    metadata = json_data.get("weatherAlerts", {}).get("metadata", {})
+    geoloc = weather.GeoLocation(
+        name=location,
+        latitude=metadata.get("latitude"),
+        longitude=metadata.get("longitude"),
+        elevation=0,
+        country_code=metadata.get("language").split("-")[-1],
+        timezone=json_data.get("location"),
+    )
 
     weather_answer = EngineResults.types.WeatherAnswer(
         current=_weather_data(geoloc, json_data["currentWeather"]),

searx/engines/duckduckgo_web.py

@@ -14,11 +14,12 @@ can't build it ourselves and must scrape it from the HTML pages.
 """
 
 import typing as t
+import re
 
-from urllib.parse import quote_plus
+from urllib.parse import quote_plus, urljoin
 from lxml import html
 
-from searx.utils import html_to_text, gen_useragent, extract_text, eval_xpath
+from searx.utils import html_to_text, extract_text, eval_xpath
 from searx.result_types import EngineResults
 from searx.enginelib import EngineCache
 from searx.network import get
@@ -38,7 +39,6 @@ about = {
 # engine dependent config
 categories = ["general"]
 paging = True
-_HTTP_User_Agent: str = gen_useragent()
 
 base_url = "https://duckduckgo.com"
 
@@ -73,6 +73,8 @@ def _fetch_first_page_link(
     resp = get(
         url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
         headers=headers,
+        impersonate="firefox",
+        default_headers=False,
         timeout=2,
     )
 
@@ -96,6 +98,43 @@ def _cache_key(query: str, pageno: int) -> str:
     return f"nextpage_url|{query}|{pageno}"
 
 
+def _solve_jsa(resp: "SXNG_Response") -> "SXNG_Response":
+    """Duckduckgo sometimes issues a challenge instead of json."""
+
+    # length that a real browser would report for where the broken snippet is
+    html_len = {
+        "<p><div></p><p></div": 32,
+        "<li><div></li><li></div": 29,
+        "<div><div></div><div></div": 33,
+        "<br><div></br><br></div": 23,
+    }
+
+    js = resp.text or ""
+    jsa_match = re.search(r"let jsa = (\d+);.*?DDG\.deep\.initialize\('([^']+)'", js, re.S)
+    if not jsa_match:
+        return resp
+
+    js_functions = dict(re.findall(r"let (\w+) = function\(num\) \{([^}]*)\};", js))
+    jsa = int(jsa_match.group(1))
+    try:
+        for name in re.findall(r"jsa = (\w+)\(jsa\);", js):
+            body = js_functions[name]
+            mul = re.search(r"num \* (\d+)", body)
+            jsa = jsa * int(mul.group(1)) if mul else jsa + html_len[re.search(r"`([^`]+)`", body).group(1)]
+    except (KeyError, AttributeError):
+        return resp
+
+    params = resp.search_params
+    follow = get(
+        urljoin("https://links.duckduckgo.com", jsa_match.group(2) + str(jsa)),
+        headers=params["headers"],
+        impersonate="firefox",
+        default_headers=False,
+    )
+    follow.search_params = params
+    return follow
+
+
 def request(query: str, params: "OnlineParams") -> None:
 
     if len(query) >= 500:
@@ -103,25 +142,15 @@ def request(query: str, params: "OnlineParams") -> None:
         params["url"] = None
         return
 
-    headers = params["headers"]
-
-    # The vqd value is generated from the query and the UA header.  To be able
-    # to reuse the vqd value, the UA header must be static.
-    headers["User-Agent"] = _HTTP_User_Agent
-    headers["Accept"] = "*/*"
-    headers["Referer"] = f"{base_url}/"
-    headers["Host"] = "duckduckgo.com"
-
-    # Sec-Fetch headers are required to not get blocked when sending a Firefox user agent
-    headers["Sec-Fetch-Dest"] = "script"
-    headers["Sec-Fetch-Mode"] = "no-cors"
-    headers["Sec-Fetch-Site"] = "same-site"
+    # firefox TLS only
+    params["impersonate"] = "firefox"
+    params["default_headers"] = False
 
     api_url = ""
     if params["pageno"] > 1:
         api_url = CACHE.get(_cache_key(query, params["pageno"]))
     else:
-        api_url = _fetch_first_page_link(query, headers)
+        api_url = _fetch_first_page_link(query, params["headers"])
 
     if not api_url:
         params["url"] = None
@@ -129,14 +158,27 @@ def request(query: str, params: "OnlineParams") -> None:
 
     params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
 
+    # loads as a script
+    headers = params["headers"]
+    headers["Accept"] = "*/*"
+    headers["Sec-Fetch-Dest"] = "script"
+    headers["Sec-Fetch-Mode"] = "no-cors"
+    headers["Sec-Fetch-Site"] = "same-site"
+    headers["Referer"] = f"{base_url}/"
+
     # TODO: support safesearch, timerange and engine traits  # pylint:disable=fixme
 
 
 def response(resp: "SXNG_Response"):
     res = EngineResults()
-    res_json = resp.json()
 
-    for result in res_json["results"]:
+    # check if ddg returns a challenge
+    # e.g. 'site:github.com searxng'
+    if "let jsa =" in (resp.text or ""):
+        resp = _solve_jsa(resp)
+
+    results = resp.json()["results"]
+    for result in results:
         if "u" not in result:
             continue
 
@@ -144,13 +186,13 @@ def response(resp: "SXNG_Response"):
             res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
         )
 
-    # link to next page
-    next_page_path = res_json["results"][-1].get("n")
-    if next_page_path:
-        CACHE.set(
-            _cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
-            base_url + next_page_path,
-            expire=60 * 60,
-        )
+    if results:
+        next_page_path = results[-1].get("n")
+        if next_page_path:
+            CACHE.set(
+                _cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
+                base_url + next_page_path,
+                expire=60 * 60,
+            )
 
     return res

searx/engines/elasticsearch.py

@@ -41,6 +41,7 @@ authentication configured to read from ``my-index`` index.
 
 """
 
+import typing as t
 from json import loads, dumps
 from searx.exceptions import SearxEngineAPIException
 from searx.result_types import EngineResults
@@ -68,7 +69,7 @@ show_metadata = False
 page_size = 10
 
 
-def init(engine_settings):
+def setup(engine_settings: dict[str, t.Any]) -> bool | None:
     if 'query_type' in engine_settings and engine_settings['query_type'] not in _available_query_types:
         raise ValueError('unsupported query type', engine_settings['query_type'])
 

searx/engines/europepmc.py

@@ -0,0 +1,150 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""`Europe PMC`_ provides comprehensive access to life sciences literature from
+trusted sources.  With Europe PMC you can search and read millions of
+publications, preprints and other documents enriched with links to supporting
+data, reviews, protocols, and other relevant resources.
+
+.. _Europe PMC: https://europepmc.org/
+
+Configuration
+=============
+
+.. code:: yaml
+
+   - name: europepmc
+     engine: europepmc
+     shortcut: epmc
+
+Implementations
+===============
+
+"""
+
+import typing as t
+
+from datetime import datetime
+from urllib.parse import urlencode
+
+from dateutil.parser import isoparse
+
+from searx.enginelib import EngineCache
+from searx.result_types import EngineResults
+from searx.utils import html_to_text
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
+
+
+about = {
+    "website": "https://europepmc.org/",
+    "wikidata_id": "Q5412157",
+    "official_api_documentation": "https://europepmc.org/RestfulWebService",
+    "use_official_api": True,
+    "require_api_key": False,
+    "results": "JSON",
+}
+
+categories = ["science", "scientific publications"]
+paging = True
+
+# engine dependent config
+search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
+article_url = "https://europepmc.org/article/"
+
+page_size = 20
+
+CACHE: EngineCache
+"""Cache for storing the pagination cursor."""
+
+
+def setup(engine_settings: dict[str, t.Any]):
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])
+
+
+def _cache_key(query: str, page: int) -> str:
+    return f"{query}|{page}"
+
+
+def request(query: str, params: "OnlineParams") -> None:
+    args = {
+        "query": query,
+        "format": "json",
+        "resultType": "core",
+        "pageSize": page_size,
+    }
+
+    if params["pageno"] > 1:
+        if cursor := CACHE.get(_cache_key(query, params["pageno"])):
+            args["cursorMark"] = cursor
+        else:
+            # no cached cursor for that page
+            params["url"] = None
+            return
+
+    params["url"] = f"{search_url}?{urlencode(args)}"
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
+
+    json_resp = resp.json()
+
+    # store pagination cursor for loading next pages in cache
+    if next_cursor := json_resp.get("nextCursorMark"):
+        next_page = resp.search_params["pageno"] + 1
+        query = resp.search_params["query"]
+        CACHE.set(_cache_key(query, next_page), next_cursor)
+
+    all_results = json_resp.get("resultList", {}).get("result", [])
+
+    for item in all_results:
+        source = item.get("source", "")
+        identifier = item.get("id", "")
+        url = f"{article_url}{source}/{identifier}" if source and identifier else ""
+
+        journal_info: dict[str, t.Any] = item.get("journalInfo", {})
+        journal: dict[str, t.Any] = journal_info.get("journal", {})
+
+        res.add(
+            res.types.Paper(
+                url=url,
+                title=html_to_text(item.get("title", "")),
+                content=html_to_text(item.get("abstractText", "")),
+                journal=journal.get("title", ""),
+                issn=[journal.get("issn", "")],
+                authors=_get_authors(item),
+                doi=item.get("doi", ""),
+                publishedDate=_get_published_date(item),
+                type=", ".join((item.get("pubTypeList", {})).get("pubType", [])),
+                pdf_url=_get_pdf_url(item),
+                html_url=url,
+            )
+        )
+
+    return res
+
+
+def _get_authors(item: dict[str, t.Any]) -> list:
+    """Extract the list of authors from the item."""
+    if authors := item.get("authorString", None):
+        authors = [author.strip().rstrip(".") for author in authors.split(",") if author.strip()]
+    else:
+        authors = []
+    return authors
+
+
+def _get_pdf_url(item: dict[str, t.Any]) -> str:
+    """Extract the PDF URL in case it is open access."""
+    for url_info in (item.get("fullTextUrlList", {})).get("fullTextUrl", []):
+        if url_info.get("documentStyle") == "pdf" and url_info.get("availabilityCode") == "OA":
+            return url_info.get("url", "")
+    return ""
+
+
+def _get_published_date(item: dict[str, t.Any]) -> datetime | None:
+    """Extract the published date from the item and convert it to a datetime object."""
+    if unformatted_date := item.get("firstPublicationDate"):
+        return isoparse(unformatted_date)
+    return None

searx/engines/exaapi.py

@@ -78,7 +78,7 @@ content_max_characters: int = 500
 """Maximum characters for the requested content."""
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if not api_key:
         raise SearxEngineAPIException("No API key provided")
     if not 1 <= results_per_page <= 100:

searx/engines/fireball.py

@@ -47,7 +47,7 @@ CACHE_VALID_DURATION = 30 * 24 * 3600  # one month, same as website
 """Duration how long settings cookies are valid."""
 
 
-def init(engine_settings: dict[str, t.Any]):
+def setup(engine_settings: dict[str, t.Any]) -> bool | None:
     global CACHE  # pylint: disable=global-statement
     CACHE = EngineCache(engine_settings["name"])
 

searx/engines/gitea.py

@@ -38,6 +38,7 @@ Implementation
 
 """
 
+import typing as t
 from urllib.parse import urlencode
 from dateutil import parser
 
@@ -77,7 +78,7 @@ page_size: int = 10
 """Maximum number of results per page (default 10)."""
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if not base_url:
         raise ValueError('gitea engine: base_url is unset')
 

searx/engines/google.py

@@ -9,12 +9,15 @@ engines:
 - :ref:`google scholar engine`
 - :ref:`google autocomplete`
 
+This implementation uses Nokia user agents to request an XML layout from Google.
+The normal web version requires executing JavaScript to load the results and
+therefore is currently not used here.  See `Google discussion`_ for more
+information on that topic.
+
+.. _Google discussion: https://github.com/searxng/searxng/issues/6359
 """
 
 import random
-import re
-import string
-import time
 import typing as t
 from urllib.parse import unquote, urlencode
 
@@ -44,16 +47,16 @@ about = {
     "official_api_documentation": "https://developers.google.com/custom-search/",
     "use_official_api": False,
     "require_api_key": False,
-    "results": "HTML",
+    "results": "XML",
 }
 
 # engine dependent config
 categories = ["general", "web"]
 paging = True
 max_page = 50
-"""`Google max 50 pages`_
+"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
 
-.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
+.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
 """
 time_range_support = True
 language_support = True
@@ -64,38 +67,23 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
 # Filter results. 0: None, 1: Moderate, 2: Strict
 filter_mapping = {0: "off", 1: "medium", 2: "high"}
 
+# https://github.com/searxng/searxng/issues/6359
+nokia_useragents = (
+    "Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
+    "Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
+    "Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+)
+
+
 # specific xpath variables
 # ------------------------
 
 # Suggestions are links placed in a *card-section*, we extract only the text
 # from the links not the links itself.
-suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
-
-
-_arcid_range = string.ascii_letters + string.digits + "_-"
-_arcid_random: tuple[str, int] | None = None
-
-
-def ui_async(start: int) -> str:
-    """Format of the response from UI's async request.
-
-    - ``arc_id:<...>,use_ac:true,_fmt:prog``
-
-    The arc_id is random generated every hour.
-    """
-    global _arcid_random  # pylint: disable=global-statement
-
-    use_ac = "use_ac:true"
-    # _fmt:html returns a HTTP 500 when user search for celebrities like
-    # '!google natasha allegri' or '!google chris evans'
-    _fmt = "_fmt:prog"
-
-    # create a new random arc_id every hour
-    if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
-        _arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
-    arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
-
-    return ",".join([arc_id, use_ac, _fmt])
+suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
 
 
 def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
@@ -127,19 +115,11 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
             A instance of :py:obj:`babel.core.Locale` build from the
             ``searxng_locale`` value.
 
-        subdomain:
-            Google subdomain :py:obj:`google_domains` that fits to the country
-            code.
-
         params:
             Py-Dictionary with additional request arguments (can be passed to
             :py:func:`urllib.parse.urlencode`).
 
             - ``hl`` parameter: specifies the interface language of user interface.
-            - ``lr`` parameter: restricts search results to documents written in
-              a particular language.
-            - ``cr`` parameter: restricts search results to documents
-              originating in a particular country.
             - ``ie`` parameter: sets the character encoding scheme that should
               be used to interpret the query string ('utf8').
             - ``oe`` parameter: sets the character encoding scheme that should
@@ -156,7 +136,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     ret_val: dict[str, t.Any] = {
         "language": None,
         "country": None,
-        "subdomain": None,
         "params": {},
         "headers": {},
         "cookies": {},
@@ -169,7 +148,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     except babel.core.UnknownLocaleError:
         locale = None
 
-    eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
+    eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
     lang_code = eng_lang.split("_")[-1]  # lang_zh-TW --> zh-TW / lang_en --> en
     country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
 
@@ -184,7 +163,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     ret_val["language"] = eng_lang
     ret_val["country"] = country
     ret_val["locale"] = locale
-    ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
 
     # hl parameter:
     #   The hl parameter specifies the interface language (host language) of
@@ -223,9 +201,11 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
 
     # specify a region (country) only if a region is given in the selected
     # locale --> https://github.com/searxng/searxng/issues/2672
-    ret_val["params"]["cr"] = ""
-    if len(sxng_locale.split("-")) > 1:
-        ret_val["params"]["cr"] = "country" + country
+
+    if country is not None:
+        ret_val["params"]["cr"] = ""
+        if len(sxng_locale.split("-")) > 1:
+            ret_val["params"]["cr"] = "country" + country
 
     # gl parameter: (mandatory by Google News)
     #   The gl parameter value is a two-letter country code. For WebSearch
@@ -300,88 +280,78 @@ def detect_google_sorry(resp: "SXNG_Response"):
         raise SearxEngineCaptchaException()
 
 
-def request(query: str, params: "OnlineParams") -> None:
-    """Google search request"""
-    # pylint: disable=line-too-long
-    start = (params["pageno"] - 1) * 10
-    google_info = get_google_info(params, traits)
-
-    # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
-    query_url = (
-        "https://"
-        + google_info["subdomain"]
-        + "/search"
-        + "?"
-        + urlencode(
-            {
-                "q": query,
-                **google_info["params"],
-                "filter": "0",
-                "start": start,
-                # 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
-                # 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
-                # 'cs' : 1,
-                # 'sa': 'N',
-                # 'yv': 3,
-                # 'prmd': 'vin',
-                # 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
-                # 'sa': 'N',
-                # 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
-                # formally known as use_mobile_ui
-                # "asearch": "arc",
-                # "async": str_async,
-            }
-        )
-    )
-
-    if params["time_range"] in time_range_dict:
-        query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
-    if params["safesearch"]:
-        query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
-    params["url"] = query_url
-
-    params["cookies"] = google_info["cookies"]
-    params["headers"].update(google_info["headers"])
+def unwrap_google_url(raw_url: str) -> str:
+    # remove redirector from url
+    if raw_url.startswith("/url?q="):
+        return unquote(raw_url[7:].split("&sa=U")[0])
+    return raw_url
 
 
-# regex match to get image map that is found inside the returned javascript:
-# (function(){var s='...';var i=['...'] ...}
-RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
-
-
-def parse_url_images(text: str):
-    data_image_map = {}
-
-    for image_url, img_id in RE_DATA_IMAGE.findall(text):
-        data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
-    logger.debug("data:image objects --> %s", list(data_image_map.keys()))
-    return data_image_map
-
-
-def response(resp: "SXNG_Response"):
-    """Get response from google's search request"""
-    # pylint: disable=too-many-branches, too-many-statements
+def wml_dom(resp: "SXNG_Response"):
     detect_google_sorry(resp)
-    data_image_map = parse_url_images(resp.text)
+    text = resp.text
+    if text.lstrip().startswith("<?xml"):
+        text = text.split("?>", 1)[-1]
+    return html.fromstring(text)
 
+
+def google_request(
+    query: str,
+    params: "OnlineParams",
+    extra_args: dict[str, t.Any] | None = None,
+    *,
+    eng_traits: EngineTraits | None = None,
+    use_time_range: bool = True,
+    use_safesearch: bool = True,
+    safesearch_map: dict[int, str] | None = None,
+    use_locales: bool = True,
+) -> None:
+    google_info = get_google_info(params, eng_traits or traits)
+    if not use_locales:
+        google_info["params"].pop("lr")
+        google_info["params"].pop("cr")
+
+    start = (params["pageno"] - 1) * 10
+    args: dict[str, t.Any] = {
+        "q": query,
+        "sca_esv": "1",
+        **google_info["params"],
+        **(extra_args or {}),
+    }
+    if start:
+        args["start"] = start
+    if use_time_range and params["time_range"] in time_range_dict:
+        args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
+    if use_safesearch and params["safesearch"]:
+        args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
+
+    params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
+    params["headers"]["User-Agent"] = random.choice(nokia_useragents)
+    params["impersonate"] = "chrome99_android"
+
+
+def request(query: str, params: "OnlineParams") -> None:
+    google_request(query, params)
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
     results = EngineResults()
-
-    # convert the text to dom
-    dom = html.fromstring(resp.text)
+    dom = wml_dom(resp)
 
     # parse results
-    for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
-        # pylint: disable=too-many-nested-blocks
+    for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
 
         try:
-            title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
+            title_tag = eval_xpath_getindex(
+                result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
+            )
             if title_tag is None:
                 # this not one of the common google results *section*
                 logger.debug("ignoring item from the result_xpath list: missing title")
                 continue
             title = extract_text(title_tag)
 
-            raw_url = result.get("href")
+            raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
             if raw_url is None:
                 logger.debug(
                     'ignoring item from the result_xpath list: missing url of title "%s"',
@@ -389,30 +359,19 @@ def response(resp: "SXNG_Response"):
                 )
                 continue
 
-            if raw_url.startswith('/url?q='):
-                url = unquote(raw_url[7:].split("&sa=U")[0])  # remove the google redirector
-            else:
-                url = raw_url
-
-            content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
-            for item in content_nodes:
-                for script in item.xpath(".//script"):
-                    script.getparent().remove(script)
-
-            content = extract_text(content_nodes[0])
-
-            # Images that are NOT the favicon
-            xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
-
-            thumbnail = None
-            if xpath_image is not None:
-                thumbnail = xpath_image.get("src")
-                if thumbnail.startswith("data:image"):
-                    img_id = xpath_image.get("id")
-                    if img_id:
-                        thumbnail = data_image_map.get(img_id)
-
-            results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
+            url = unwrap_google_url(raw_url)
+            content = extract_text(
+                eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
+            )
+            thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
+            results.add(
+                results.types.MainResult(
+                    url=url,
+                    title=title or "",
+                    content=content or "",
+                    thumbnail=thumbnail or "",
+                )
+            )
 
         except Exception as e:  # pylint: disable=broad-except
             logger.error(e, exc_info=True)
@@ -420,10 +379,8 @@ def response(resp: "SXNG_Response"):
 
     # parse suggestion
     for suggestion in eval_xpath_list(dom, suggestion_xpath):
-        # append suggestion
-        results.append({"suggestion": extract_text(suggestion)})
+        results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
 
-    # return results
     return results
 
 
@@ -456,14 +413,12 @@ skip_countries = [
 ]
 
 
-def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
+def fetch_traits(engine_traits: EngineTraits):
     """Fetch languages from Google."""
     # pylint: disable=import-outside-toplevel, too-many-branches
 
     from searx.network import get  # see https://github.com/searxng/searxng/issues/762
 
-    engine_traits.custom["supported_domains"] = {}
-
     resp = get("https://www.google.com/preferences", timeout=5)
     if not resp.ok:
         raise RuntimeError("Response from Google preferences is not OK.")
@@ -514,22 +469,3 @@ def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
 
     # alias regions
     engine_traits.regions["zh-CN"] = "HK"
-
-    # supported domains
-
-    if add_domains:
-        resp = get("https://www.google.com/supported_domains", timeout=5)
-        if not resp.ok:
-            raise RuntimeError("Response from Google supported domains is not OK.")
-
-        for domain in resp.text.split():
-            domain = domain.strip()
-            if not domain or domain in [
-                ".google.com",
-            ]:
-                continue
-            region = domain.split(".")[-1].upper()
-            engine_traits.custom["supported_domains"][region] = "www" + domain
-            if region == "HK":
-                # There is no google.cn, we use .com.hk for zh-CN
-                engine_traits.custom["supported_domains"]["CN"] = "www" + domain

searx/engines/google_cse.py

@@ -30,6 +30,7 @@ about = {
 
 categories = ["general", "web"]
 paging = True
+enable_http3 = True
 max_page = 5
 page_size = 20
 time_range_support = True
@@ -95,12 +96,11 @@ def request(query: str, params: "OnlineParams") -> None:
     token = _cse_token()
 
     google_info = get_google_info(params, traits)
-    info: dict[str, str] = google_info["params"]
 
     args = {
         "rsz": "filtered_cse",
         "num": str(page_size),
-        "hl": info["hl"],
+        "hl": google_info["params"]["hl"],
         "cselibv": token["cselibv"],
         "cx": CX,
         "q": query,
@@ -114,10 +114,6 @@ def request(query: str, params: "OnlineParams") -> None:
         start_date, end_date = _get_start_and_end_date_str(params["time_range"])
         args["sort"] = f"date:r:{start_date}:{end_date}"
 
-    if info.get("lr"):
-        args["lr"] = info["lr"]
-    if info.get("cr"):
-        args["cr"] = info["cr"]
     if google_info["country"] not in (None, "ZZ"):
         args["gl"] = google_info["country"]
     if token["exp"]:

searx/engines/google_images.py

@@ -1,122 +1,76 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""This is the implementation of the Google Images engine using the internal
-Google API used by the Google Go Android app.
+"""Google Images: see :py:obj:`searx.engines.google`."""
 
-This internal API offer results in
-
-- JSON (``_fmt:json``)
-- Protobuf_ (``_fmt:pb``)
-- Protobuf_ compressed? (``_fmt:pc``)
-- HTML (``_fmt:html``)
-- Protobuf_ encoded in JSON (``_fmt:jspb``).
-
-.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
-"""
-
-from urllib.parse import urlencode
-from json import loads
+import typing as t
+from urllib.parse import parse_qs, unquote, urlparse
 
 from searx.engines.google import fetch_traits  # pylint: disable=unused-import
-from searx.engines.google import (
-    get_google_info,
-    time_range_dict,
-    detect_google_sorry,
-)
+from searx.engines.google import google_request, wml_dom
+from searx.result_types import EngineResults
+from searx.utils import eval_xpath_list
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
 
 # about
 about = {
-    "website": 'https://images.google.com',
-    "wikidata_id": 'Q521550',
-    "official_api_documentation": 'https://developers.google.com/custom-search',
+    "website": "https://images.google.com",
+    "wikidata_id": "Q521550",
+    "official_api_documentation": "https://developers.google.com/custom-search",
     "use_official_api": False,
     "require_api_key": False,
-    "results": 'JSON',
+    "results": "XML",
 }
 
 # engine dependent config
-categories = ['images', 'web']
+categories = ["images", "web"]
 paging = True
+enable_http3 = True
 max_page = 50
-"""`Google max 50 pages`_
+"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
 
-.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
+.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
 """
 
 time_range_support = True
 language_support = True
 safesearch = True
 
-filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
+filter_mapping = {0: "images", 1: "active", 2: "active"}
 
 
-def request(query, params):
-    """Google-Image search request"""
-
-    google_info = get_google_info(params, traits)
-
-    query_url = (
-        'https://'
-        + google_info['subdomain']
-        + '/search'
-        + '?'
-        + urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
-        # don't urlencode this because wildly different AND bad results
-        # pagination uses Zero-based numbering
-        + f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
+def request(query: str, params: "OnlineParams") -> None:
+    google_request(
+        query,
+        params,
+        {"tbm": "isch"},
+        eng_traits=traits,
+        safesearch_map=filter_mapping,
+        use_locales=False,
     )
 
-    if params['time_range'] in time_range_dict:
-        query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
-    if params['safesearch']:
-        query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
-    params['url'] = query_url
-    params['cookies'] = google_info['cookies']
-    params['headers'].update(google_info['headers'])
-    # this ua will allow getting ~50 results instead of 10. #1641
-    params['headers']['User-Agent'] = (
-        'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
-    )
 
-    return params
+def response(resp: "SXNG_Response") -> EngineResults:
+    results = EngineResults()
+    dom = wml_dom(resp)
 
-
-def response(resp):
-    """Get response from google's search request"""
-    results = []
-
-    detect_google_sorry(resp)
-
-    json_start = resp.text.find('{"ischj":')
-    json_data = loads(resp.text[json_start:])
-
-    for item in json_data["ischj"].get("metadata", []):
-        result_item = {
-            'url': item["result"]["referrer_url"],
-            'title': item["result"]["page_title"],
-            'content': item["text_in_grid"]["snippet"],
-            'source': item["result"]["site_title"],
-            'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
-            'img_src': item["original_image"]["url"],
-            'thumbnail_src': item["thumbnail"]["url"],
-            'template': 'images.html',
-        }
-
-        author = item["result"].get('iptc', {}).get('creator')
-        if author:
-            result_item['author'] = ', '.join(author)
-
-        copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
-        if copyright_notice:
-            result_item['source'] += ' | ' + copyright_notice
-
-        freshness_date = item["result"].get("freshness_date")
-        if freshness_date:
-            result_item['source'] += ' | ' + freshness_date
-
-        file_size = item.get('gsa', {}).get('file_size')
-        if file_size:
-            result_item['source'] += ' (%s)' % file_size
-
-        results.append(result_item)
+    for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
+        qs = parse_qs(urlparse(link.get("href", "")).query)
+        img_src = qs.get("imgurl", [""])[0]
+        url = qs.get("imgrefurl", [""])[0]
+        if not img_src or not url:
+            continue
+        width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
+        tbnid = qs.get("tbnid", [""])[0]
+        results.add(
+            results.types.Image(
+                url=url,
+                title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
+                img_src=img_src,
+                thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
+                resolution=f"{width} x {height}" if width and height else "",
+            )
+        )
 
     return results

searx/engines/google_news.py

@@ -1,324 +1,91 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""This is the implementation of the Google News engine.
+"""Google News: see :py:obj:`searx.engines.google`."""
 
-Google News has a different region handling compared to Google WEB.
-
-- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
-- the hl_ argument has to be set correctly (and different to Google WEB)
-- the gl_ argument is mandatory
-
-If one of this argument is not set correctly, the request is redirected to
-CONSENT dialog::
-
-  https://consent.google.com/m?continue=
-
-The google news API ignores some parameters from the common :ref:`google API`:
-
-- num_ : the number of search results is ignored / there is no paging all
-  results for a query term are in the first response.
-- save_ : is ignored / Google-News results are always *SafeSearch*
-
-.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
-.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
-.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
-.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
-"""
 import typing as t
 
-import json
-import base64
-from urllib.parse import urlencode
-from lxml import html
-import babel
-
-from searx import locales
+from searx.engines.google import fetch_traits  # pylint: disable=unused-import
+from searx.engines.google import google_request, unwrap_google_url, wml_dom
+from searx.result_types import EngineResults
 from searx.utils import (
-    eval_xpath,
-    eval_xpath_list,
     eval_xpath_getindex,
+    eval_xpath_list,
     extract_text,
 )
 
-from searx.engines.google import fetch_traits as _fetch_traits  # pylint: disable=unused-import
-from searx.engines.google import (
-    get_google_info,
-    detect_google_sorry,
-)
-from searx.enginelib.traits import EngineTraits
-
-from searx.result_types import EngineResults
-
 if t.TYPE_CHECKING:
     from searx.extended_types import SXNG_Response
     from searx.search.processors import OnlineParams
 
 # about
 about = {
-    "website": "https://news.google.com",
+    "website": "https://www.google.com",
     "wikidata_id": "Q12020",
     "official_api_documentation": "https://developers.google.com/custom-search",
     "use_official_api": False,
     "require_api_key": False,
-    "results": "HTML",
+    "results": "XML",
 }
 
 # engine dependent config
 categories = ["news"]
-paging = False
+paging = True
+max_page = 50
+"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
+
+.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
+"""
 time_range_support = False
 language_support = True
-
-# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
-# False here.
-#
-#  safesearch : results are identical for safesearch=0 and safesearch=2
-safesearch = True
-base_url: str = "https://news.google.com"
+safesearch = False
 
 
 def request(query: str, params: "OnlineParams") -> None:
-    """Google-News search request"""
-
-    sxng_locale = params.get("searxng_locale", "en-US")
-    ceid: str = locales.get_engine_locale(
-        sxng_locale, traits.custom["ceid"], default="US:en"
-    )  # pyright: ignore[reportAssignmentType]
-    google_info = get_google_info(params, traits)
-    google_info["subdomain"] = "news.google.com"  # google news has only one domain
-
-    ceid_region, ceid_lang = ceid.split(":")
-    ceid_lang, ceid_suffix = (
-        ceid_lang.split(":")
-        + [
-            "",
-        ]
-    )[:2]
-
-    google_info["params"]["hl"] = ceid_lang
-
-    if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
-
-        if ceid_region.lower() == ceid_lang:
-            google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
-        else:
-            google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
-
-    elif ceid_region.lower() != ceid_lang:
-
-        if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
-            google_info["params"]["hl"] = ceid_lang
-        else:
-            google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
-
-    google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
-    google_info["params"]["gl"] = ceid_region
-
-    query_url = (
-        "https://"
-        + google_info["subdomain"]
-        + "/search?"
-        + urlencode(
-            {"q": query, **google_info["params"]},
-        )
-        # ceid includes a ':' character which must not be urlencoded
-        + ("&ceid=%s" % ceid)
+    google_request(
+        query,
+        params,
+        {"tbm": "nws"},
+        eng_traits=traits,
+        use_time_range=False,
+        use_safesearch=False,
+        use_locales=False,
     )
 
-    params["url"] = query_url
-    params["cookies"] = google_info["cookies"]
-    params["headers"].update(google_info["headers"])
+
+def _span_text(link, css_class: str):
+    return extract_text(
+        eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
+        allow_none=True,
+    )
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    """Get response from google's search request"""
-
-    res = EngineResults()
-
-    detect_google_sorry(resp)
-
-    # convert the text to dom
-    dom = html.fromstring(resp.text)
-
-    for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
-
-        url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
-        if not url:
-            continue
-        if url.startswith("./"):
-            url = base_url + url[1:]
-
-        # The real URL is often encoded in the "jslog" attribute
-        jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
-
-        # Try to extract the real URL from jslog
-        real_url: str | None = None
-        if jslog:
-            # jslog format is usually: "95014; 5:<base64>; track:click,vis".  We
-            # want the second part (index 1) after splitting by ";"
-            parts: list[str] = jslog.split(";")
-            if len(parts) > 1:
-                b64_data: str = parts[1].split(":")[-1].strip()
-                # Pad base64 if necessary
-                b64_data += "=" * (-len(b64_data) % 4)
-                decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
-                # The URL is typically the last element in the decoded array
-                if (
-                    isinstance(decoded_data, list)
-                    and isinstance(decoded_data[-1], str)
-                    and decoded_data[-1].startswith("http")
-                ):
-                    real_url = decoded_data[-1]
-        if real_url:
-            url = real_url
-        else:
-            logger.error(f"no real-url found: {url}")
+    results = EngineResults()
+    seen = set()
+    for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
+        href = link.get("href")
+        if not href:
             continue
 
-        title = extract_text(eval_xpath(result, "./h4")) or ""
+        url = unwrap_google_url(href)
+        if url in seen or "google.com/search" in url:
+            continue
 
-        # The pub_date is mostly a string like 'yesterday', not a real timezone
-        # date or time.  Therefore we can't use publishedDate and place the
-        # *pub* sting into the content.
+        title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
+        if not title:
+            continue
 
-        pub_date = extract_text(eval_xpath(result, ".//time"))
-        pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
-        content = " / ".join([x for x in [pub_origin, pub_date] if x])
+        source = _span_text(link, "dXDvrc")
+        pub_date = _span_text(link, "YVIcad")
+        thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
 
-        thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
-        if thumbnail and thumbnail.startswith("/"):
-            thumbnail = base_url + thumbnail
-
-        res.add(
-            res.types.MainResult(
+        seen.add(url)
+        results.add(
+            results.types.MainResult(
                 url=url,
                 title=title,
-                content=content,
-                thumbnail=thumbnail,
+                content=" / ".join(x for x in [source, pub_date] if x),
+                thumbnail=thumbnail or "",
             )
         )
 
-    return res
-
-
-ceid_list = [
-    "AE:ar",
-    "AR:es-419",
-    "AT:de",
-    "AU:en",
-    "BD:bn",
-    "BE:fr",
-    "BE:nl",
-    "BG:bg",
-    "BR:pt-419",
-    "BW:en",
-    "CA:en",
-    "CA:fr",
-    "CH:de",
-    "CH:fr",
-    "CL:es-419",
-    "CN:zh-Hans",
-    "CO:es-419",
-    "CU:es-419",
-    "CZ:cs",
-    "DE:de",
-    "EE:et",
-    "EG:ar",
-    "ES:ca",
-    "ES:es",
-    "ET:en",
-    "FI:fi",
-    "FR:fr",
-    "GB:en",
-    "GH:en",
-    "GR:el",
-    "HK:zh-Hant",
-    "HU:hu",
-    "ID:en",
-    "ID:id",
-    "IE:en",
-    "IL:en",
-    "IL:he",
-    "IN:bn",
-    "IN:en",
-    "IN:gu",
-    "IN:hi",
-    "IN:ml",
-    "IN:mr",
-    "IN:pa",
-    "IN:ta",
-    "IN:te",
-    "IT:it",
-    "JP:ja",
-    "KE:en",
-    "KR:ko",
-    "LB:ar",
-    "LT:lt",
-    "LV:en",
-    "LV:lv",
-    "MA:fr",
-    "MY:en",
-    "MY:ms",
-    "NA:en",
-    "NG:en",
-    "NL:nl",
-    "NO:no",
-    "NZ:en",
-    "PH:en",
-    "PK:en",
-    "PL:pl",
-    "RO:ro",
-    "RS:sr",
-    "RU:ru",
-    "SA:ar",
-    "SE:sv",
-    "SG:en",
-    "SI:sl",
-    "SK:sk",
-    "SN:fr",
-    "TH:th",
-    "TR:tr",
-    "TZ:en",
-    "UA:ru",
-    "UA:uk",
-    "UG:en",
-    "US:en",
-    "VN:vi",
-    "ZA:en",
-    "ZW:en",
-]
-"""List of region/language combinations supported by Google News.  Values of the
-``ceid`` argument of the Google News REST API."""
-
-
-_skip_values = [
-    "ET:en",  # english (ethiopia)
-    "ID:en",  # english (indonesia)
-    "LV:en",  # english (latvia)
-]
-
-_ceid_locale_map = {"NO:no": "nb-NO"}
-
-
-def fetch_traits(engine_traits: EngineTraits):
-    _fetch_traits(engine_traits, add_domains=False)
-
-    engine_traits.custom["ceid"] = {}
-
-    for ceid in ceid_list:
-        if ceid in _skip_values:
-            continue
-
-        region, lang = ceid.split(":")
-        x = lang.split("-")
-        if len(x) > 1:
-            if x[1] not in ["Hant", "Hans"]:
-                lang = x[0]
-
-        sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
-        try:
-            locale = babel.Locale.parse(sxng_locale, sep="-")
-        except babel.UnknownLocaleError:
-            print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
-            continue
-
-        engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid
+    return results

searx/engines/google_play.py

@@ -20,6 +20,7 @@ about = {
 }
 
 play_categ = None  # apps|movies
+enable_http3 = True
 base_url = 'https://play.google.com'
 search_url = base_url + "/store/search?{query}&c={play_categ}"
 

searx/engines/google_scholar.py

@@ -27,7 +27,7 @@ import typing as t
 from urllib.parse import urlencode
 from datetime import datetime
 from lxml import html
-import httpx
+from curl_cffi.requests.exceptions import TooManyRedirects
 
 from searx.utils import (
     eval_xpath,
@@ -63,6 +63,7 @@ about = {
 # engine dependent config
 categories = ["science", "scientific publications"]
 paging = True
+enable_http3 = True
 max_page = 50
 """`Google max 50 pages`_
 
@@ -77,8 +78,6 @@ def request(query: str, params: "OnlineParams") -> None:
     """Google-Scholar search request"""
 
     google_info = get_google_info(params, traits)
-    # subdomain is: scholar.google.xy
-    google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
 
     args = {
         "q": query,
@@ -89,7 +88,7 @@ def request(query: str, params: "OnlineParams") -> None:
     }
     args.update(time_range_args(params))
 
-    params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
+    params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
     params["cookies"] = google_info["cookies"]
     params["headers"].update(google_info["headers"])
 
@@ -104,7 +103,7 @@ def response(resp: "SXNG_Response") -> EngineResults:  # pylint: disable=too-man
             raise SearxEngineAccessDeniedException(
                 message="google_scholar: unusual traffic detected",
             )
-        raise httpx.TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
+        raise TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
 
     res = EngineResults()
     dom = html.fromstring(resp.text)

searx/engines/google_videos.py

@@ -1,185 +1,85 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""This is the implementation of the Google Videos engine.
+"""Google Videos: see :py:obj:`searx.engines.google`."""
 
-.. admonition:: Content-Security-Policy (CSP)
-
-   This engine needs to allow images from the `data URLs`_ (prefixed with the
-   ``data:`` scheme)::
-
-     Header set Content-Security-Policy "img-src 'self' data: ;"
-
-.. _data URLs:
-   https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
-"""
-import re
-from urllib.parse import urlencode, urlparse, parse_qs, unquote
-from lxml import html
-
-from searx.utils import (
-    eval_xpath_list,
-    eval_xpath_getindex,
-    extract_text,
-)
+import typing as t
 
 from searx.engines.google import fetch_traits  # pylint: disable=unused-import
-from searx.engines.google import (
-    get_google_info,
-    time_range_dict,
-    filter_mapping,
-    suggestion_xpath,
-    detect_google_sorry,
-    ui_async,
+from searx.engines.google import google_request, unwrap_google_url, wml_dom
+from searx.result_types import EngineResults
+from searx.utils import (
+    eval_xpath_getindex,
+    eval_xpath_list,
+    extract_text,
+    parse_duration_string,
 )
-from searx.utils import get_embeded_stream_url
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
 
 # about
 about = {
-    "website": 'https://www.google.com',
-    "wikidata_id": 'Q219885',
-    "official_api_documentation": 'https://developers.google.com/custom-search',
+    "website": "https://www.google.com",
+    "wikidata_id": "Q219885",
+    "official_api_documentation": "https://developers.google.com/custom-search",
     "use_official_api": False,
     "require_api_key": False,
-    "results": 'HTML',
+    "results": "XML",
 }
 
 # engine dependent config
-categories = ['videos', 'web']
+categories = ["videos", "web"]
 paging = True
 max_page = 50
+"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
+
+.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
+"""
 language_support = True
 time_range_support = True
 safesearch = True
 
 
-# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
-# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
-RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
-
-
-def parse_data_images(text: str):
-    data_image_map = {}
-
-    for img_id, data_image in RE_DATA_IMAGE.findall(text):
-        end_pos = data_image.rfind("=")
-        if end_pos > 0:
-            data_image = data_image[: end_pos + 1]
-        data_image_map[img_id] = data_image
-    logger.debug("data:image objects --> %s", list(data_image_map.keys()))
-    return data_image_map
-
-
-def request(query, params):
-    """Google-Video search request"""
-    google_info = get_google_info(params, traits)
-    start = (params['pageno'] - 1) * 10
-
-    query_url = (
-        'https://'
-        + google_info['subdomain']
-        + '/search'
-        + "?"
-        + urlencode(
-            {
-                'q': query,
-                'tbm': "vid",
-                'start': start,
-                **google_info['params'],
-                'asearch': 'arc',
-                'async': ui_async(start),
-            }
-        )
+def request(query: str, params: "OnlineParams") -> None:
+    google_request(
+        query,
+        params,
+        {"tbm": "vid"},
+        eng_traits=traits,
+        use_locales=False,
     )
 
-    if params['time_range'] in time_range_dict:
-        query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
-    if 'safesearch' in params:
-        query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
-    params['url'] = query_url
 
-    params['cookies'] = google_info['cookies']
-    params['headers'].update(google_info['headers'])
-    return params
+def response(resp: "SXNG_Response") -> EngineResults:
+    results = EngineResults()
 
-
-def response(resp):
-    """Get response from google's search request"""
-    results = []
-
-    detect_google_sorry(resp)
-    data_image_map = parse_data_images(resp.text)
-
-    # convert the text to dom
-    dom = html.fromstring(resp.text)
-
-    result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
-
-    # parse results
-    for result in result_divs:
+    for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
         title = extract_text(
-            eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
+            eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
             allow_none=True,
         )
-        url = eval_xpath_getindex(
-            result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
-        )
-        if url and url.startswith('/url?q='):
-            url = unquote(url[7:].split('&sa=U')[0])
+        raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
+        if not title or not raw_url:
+            continue
 
-        content = extract_text(
-            eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
-        )
-        pub_info = extract_text(
-            eval_xpath_getindex(
-                result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
-            ),
-            allow_none=True,
-        )
-        # Broader XPath to find any <img> element
-        thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
-        duration = extract_text(
-            eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
-        )
-        video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
+        url = unwrap_google_url(raw_url)
+        thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
+        if "/default.jpg" in thumbnail:
+            thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
+        length = None
+        for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
+            length = parse_duration_string(extract_text(span) or "")
+            if length:
+                break
 
-        # Fallback for video_id from URL if not found via XPath
-        if not video_id and url and 'youtube.com' in url:
-            parsed_url = urlparse(url)
-            video_id = parse_qs(parsed_url.query).get('v', [None])[0]
-
-        # Handle thumbnail
-        if thumbnail and thumbnail.startswith('data:image'):
-            img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
-            if img_id and img_id in data_image_map:
-                thumbnail = data_image_map[img_id]
-            else:
-                thumbnail = None
-        if not thumbnail and video_id:
-            thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
-
-        # Handle video embed URL
-        embed_url = None
-        if video_id:
-            embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
-        elif url:
-            embed_url = get_embeded_stream_url(url)
-
-        # Only append results with valid title and url
-        if title and url:
-            results.append(
-                {
-                    'url': url,
-                    'title': title,
-                    'content': content or '',
-                    'author': pub_info,
-                    'thumbnail': thumbnail,
-                    'length': duration,
-                    'iframe_src': embed_url,
-                    'template': 'videos.html',
-                }
+        results.add(
+            results.types.MainResult(
+                url=url,
+                title=title,
+                thumbnail=thumbnail,
+                length=length,
+                template="videos.html",
             )
-
-    # parse suggestion
-    for suggestion in eval_xpath_list(dom, suggestion_xpath):
-        results.append({'suggestion': extract_text(suggestion)})
+        )
 
     return results

searx/engines/heexy.py

@@ -1,127 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""Heexy_ is a minimalist search engine that focuses on privacy.
-
-Although it also supports news and videos, these are not implemented here
-because they usually return no result to very few irrelevant ones.
-
-It seems to use Bing internally, as the image thumbnails are loaded from Bing.
-
-.. _Heexy: https://docs.heexy.org/introduction
-
-"""
-
-from urllib.parse import urlencode
-
-import typing as t
-
-from searx.enginelib import EngineCache
-from searx.network import get
-from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
-from searx.result_types import EngineResults
-from searx.utils import gen_useragent
-
-if t.TYPE_CHECKING:
-    from searx.extended_types import SXNG_Response
-    from searx.search.processors import OnlineParams
-
-about = {
-    "website": "https://heexy.org",
-    "wikidata_id": None,
-    "official_api_documentation": None,
-    "use_official_api": False,
-    "require_api_key": False,
-    "results": "JSON",
-}
-
-paging = True
-safesearch = True
-
-categories = ["general"]
-heexy_categ = "web"
-"""Category to search in. Can be either "web" or "image"."""
-
-
-base_url = "https://heexy.org"
-api_url = "https://seapi.heexy.org"
-safe_search_map = {0: "off", 1: "on", 2: "on"}
-
-CACHE: EngineCache
-"""Cache for storing the ``X-Data-Cacheft`` token (acts like an API key)."""
-
-
-def setup(engine_settings: dict[str, t.Any]) -> bool:
-    global CACHE  # pylint: disable=global-statement
-
-    if heexy_categ not in ("web", "image"):
-        raise ValueError("invalid search category: %s" % heexy_categ)
-
-    CACHE = EngineCache(engine_settings["name"])
-    return True
-
-
-def _get_api_token(query: str) -> str:
-    """The API token is independent of the search query. We just need any query
-    to obtain it initially, and don't hardcode it here to decrease chances of
-    getting blocked. The token must be passed as ``X-Data-Cacheft`` header."""
-
-    cached_token: str = CACHE.get("token")
-    if cached_token:
-        return cached_token
-
-    resp = get(
-        f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent(), "Accept-Language": "en-US,en:q=0.9"}
-    )
-    if not resp.ok:
-        raise SearxEngineAPIException("failed to obtain request token: invalid response code")
-
-    token = resp.cookies["cacheft"]
-    if not token:
-        raise SearxEngineAPIException("failed to obtain request token: no token found")
-
-    CACHE.set("token", token, expire=3 * 60)
-    return token
-
-
-def request(query: str, params: "OnlineParams") -> None:
-    args = {
-        "q": query,
-        "page": params["pageno"],
-        "safe": safe_search_map[params["safesearch"]],
-    }
-    if params["searxng_locale"] != "all":
-        args["lang"] = params["searxng_locale"].split("-")[0]
-
-    params["url"] = f"{api_url}/search/{heexy_categ}?{urlencode(args)}"
-
-    params["headers"]["Origin"] = api_url
-    params["cookies"]["cacheft"] = _get_api_token(query)
-
-
-def response(resp: "SXNG_Response"):
-    res = EngineResults()
-
-    json_resp = resp.json()
-    if not json_resp["success"]:
-        raise SearxEngineAccessDeniedException()
-
-    result: dict[str, str]
-    for result in json_resp["results"]:
-        if heexy_categ == "web":
-            res.add(
-                res.types.MainResult(
-                    url=result["url"],
-                    title=result["title"],
-                    content=result["description"],
-                )
-            )
-        elif heexy_categ == "image":
-            res.add(
-                res.types.Image(
-                    title=result["description"],
-                    url=result["url"],
-                    thumbnail_src=result["image"],
-                    img_src=result["rawImage"],
-                )
-            )
-
-    return res

searx/engines/huggingface.py

@@ -33,6 +33,7 @@ Implementations
 
 """
 
+import typing as t
 from urllib.parse import urlencode
 from datetime import datetime
 
@@ -62,7 +63,7 @@ huggingface_endpoint = 'models'
 """
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if huggingface_endpoint not in ('datasets', 'models', 'spaces'):
         raise SearxEngineAPIException(f"Unsupported Hugging Face endpoint: {huggingface_endpoint}")
 

searx/engines/invidious.py

@@ -8,6 +8,7 @@ No public instance offer a public API now
 
 """
 
+import typing as t
 import time
 import random
 from urllib.parse import quote_plus, urlparse
@@ -34,7 +35,7 @@ time_range_support = True
 base_url: list[str] | str = []
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if not base_url:
         raise ValueError("missing invidious base_url")
 

searx/engines/kagi.py

@@ -87,7 +87,7 @@ api_key = ""
 """Kagi API key. Required for using this engine."""
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if not api_key:
         raise ValueError("api_key is required for using kagi")
 

searx/engines/luxxle.py

@@ -37,21 +37,17 @@ about = {
 }
 
 categories = []
-safeseach = True
+safesearch = True
 
 base_url = "https://luxxle.com"
 
 luxxle_categ = "search"
 """Supported categories: "search", "news", "images", "videos"."""
 
-# otherwise all requests get blocked (http2-fingerprinted probably)
-enable_http2 = False
-
-
 safe_search_map = {0: "Off", 1: "Moderate", 2: "Strict"}
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if luxxle_categ not in ("search", "images", "videos", "news"):
         raise ValueError("invalid luxxle category: %s" % luxxle_categ)
 

searx/engines/marginalia.py

@@ -6,6 +6,27 @@ Lofgren .
 .. _Marginalia Search:
    https://about.marginalia-search.com/
 
+
+.. _marginalia filters:
+
+Marginalia Filters
+==================
+
+Custom filters enable server-side customization of Marginalia search results.
+Filter definitions are written in XML and scoped to an API key.  Filters can
+not be used with the public API key ``public``.  The
+`Marginalia Filter Editor`_ can be used to create custom filters with a GUI.
+Alternatively, filters can be written manually in XML.  To associate a filter
+definition with an API key, upload the XML data to the ``/filter/<NAME>`` API
+endpoint, where ``<NAME>`` is the name for the newly created filter.  For more
+information, see the `Marginalia filters announcement blogpost`_ and the
+official `Marginalia API documentation`_.
+
+.. _Marginalia Filter Editor: https://marginalia-search.com/filters
+.. _Marginalia filters announcement blogpost: https://www.marginalia.nu/log/a_127_index_filtering/
+.. _Marginalia API documentation: https://about.marginalia-search.com/article/api/
+
+
 Configuration
 =============
 
@@ -13,6 +34,10 @@ The engine has the following required settings:
 
 - :py:obj:`api_key`
 
+The engine has the following optional settings:
+
+- :py:obj:`filter_name`
+
 You can configure a Marginalia engine by:
 
 .. code:: yaml
@@ -21,6 +46,7 @@ You can configure a Marginalia engine by:
      engine: marginalia
      shortcut: mar
      api_key: ...
+     filter_name: ...
 
 Implementations
 ===============
@@ -29,6 +55,8 @@ Implementations
 
 import typing as t
 from urllib.parse import urlencode
+
+from searx.network import get
 from searx.utils import searxng_useragent
 from searx.result_types import EngineResults
 from searx.extended_types import SXNG_Response
@@ -54,6 +82,8 @@ api_key = None
    https://about.marginalia-search.com/article/api/
 
 """
+filter_name: str = ""
+"""The name of the custom filter to apply to each search."""
 
 
 class ApiSearchResult(t.TypedDict):
@@ -83,6 +113,25 @@ class ApiSearchResults(t.TypedDict):
     results: list[ApiSearchResult]
 
 
+def _marginalia_headers() -> dict[str, t.Any]:
+    return {
+        "User-Agent": searxng_useragent(),
+        "API-Key": api_key,
+    }
+
+
+def _get_filter_names() -> list[str]:
+
+    resp = get(f"{base_url}/filter", headers=_marginalia_headers())
+    if resp.ok:
+        filter_names = resp.json()
+    else:
+        filter_names = []
+    if not isinstance(filter_names, list):
+        raise TypeError("marginalia api returned invalid filter list format")
+    return filter_names
+
+
 def request(query: str, params: dict[str, t.Any]):
 
     query_params = {
@@ -91,10 +140,11 @@ def request(query: str, params: dict[str, t.Any]):
         "nsfw": min(params["safesearch"], 1),
         "query": query,
     }
+    if filter_name:
+        query_params["filter"] = filter_name
 
     params["url"] = f"{base_url}/search?{urlencode(query_params)}"
-    params["headers"]["User-Agent"] = searxng_useragent()
-    params["headers"]["API-Key"] = api_key
+    params["headers"].update(_marginalia_headers())
 
 
 def response(resp: SXNG_Response):
@@ -114,14 +164,18 @@ def response(resp: SXNG_Response):
     return res
 
 
-def init(engine_settings: dict[str, t.Any]):
+def setup(_: dict[str, t.Any]) -> bool:
 
-    _api_key = engine_settings.get("api_key")
-    if not _api_key:
+    if not api_key:
         logger.error("missing api_key: see https://about.marginalia-search.com/article/api")
         return False
 
-    if _api_key == "public":
+    if api_key == "public":
         logger.error("invalid api_key (%s): see https://about.marginalia-search.com/article/api", api_key)
+    elif filter_name:
+        filter_names: list[str] = _get_filter_names()
+        if filter_name not in filter_names:
+            logger.error(f"invalid value for filter_name: '{filter_name}'")
+            return False
 
     return True

searx/engines/mariadb_server.py

@@ -26,6 +26,8 @@ Implementations
 
 """
 
+import typing as t
+
 try:
     import mariadb  # pyright: ignore [reportMissingImports]
 except ImportError:
@@ -60,15 +62,17 @@ paging = True
 _connection = None
 
 
-def init(engine_settings):
-    global _connection  # pylint: disable=global-statement
-
+def setup(engine_settings: dict[str, t.Any]) -> bool | None:
     if 'query_str' not in engine_settings:
         raise ValueError('query_str cannot be empty')
 
     if not engine_settings['query_str'].lower().startswith('select '):
         raise ValueError('only SELECT query is supported')
 
+
+def init(_):
+    global _connection  # pylint: disable=global-statement
+
     _connection = mariadb.connect(database=database, user=username, password=password, host=host, port=port)
 
 

searx/engines/meilisearch.py

@@ -35,7 +35,7 @@ Here is a simple example to query a Meilisearch instance:
 """
 
 # pylint: disable=global-statement
-
+import typing as t
 from json import dumps
 from searx.result_types import EngineResults
 from searx.extended_types import SXNG_Response
@@ -49,7 +49,7 @@ categories = ['general']
 paging = True
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if index == '':
         raise ValueError('index cannot be empty')
 

searx/engines/mojeek.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Mojeek (general, images, news)"""
 
+import typing as t
 from datetime import datetime
 from urllib.parse import urlencode
 
@@ -51,7 +52,7 @@ region_param = "arc"
 _delta_kwargs = {"day": "days", "week": "weeks", "month": "months", "year": "years"}
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if search_type not in ("", "images", "news"):
         raise ValueError(f"Invalid search type {search_type}")
 

searx/engines/mrs.py

@@ -19,6 +19,7 @@ Implementation
 ==============
 """
 
+import typing as t
 from urllib.parse import quote_plus
 
 about = {
@@ -37,7 +38,7 @@ matrix_url = "https://matrix.to"
 page_size = 20
 
 
-def init(engine_settings):  # pylint: disable=unused-argument
+def setup(_: dict[str, t.Any]) -> bool | None:
     """The ``base_url`` must be set in the configuration, if ``base_url`` is not
     set, a :py:obj:`ValueError` is raised during initialization.
 

searx/engines/mysql_server.py

@@ -25,6 +25,8 @@ Implementations
 
 """
 
+import typing as t
+
 from searx.result_types import EngineResults
 
 try:
@@ -60,15 +62,17 @@ paging = True
 _connection = None
 
 
-def init(engine_settings):
-    global _connection  # pylint: disable=global-statement
-
+def setup(engine_settings: dict[str, t.Any]) -> bool | None:
     if 'query_str' not in engine_settings:
         raise ValueError('query_str cannot be empty')
 
     if not engine_settings['query_str'].lower().startswith('select '):
         raise ValueError('only SELECT query is supported')
 
+
+def init(_):
+    global _connection  # pylint: disable=global-statement
+
     _connection = mysql.connector.connect(
         database=database,
         user=username,

searx/engines/naver.py

@@ -2,6 +2,8 @@
 # pylint: disable=line-too-long
 """Naver for SearXNG"""
 
+import typing as t
+
 from urllib.parse import urlencode
 from lxml import html
 
@@ -16,7 +18,6 @@ from searx.utils import (
     html_to_text,
     parse_duration_string,
     js_obj_str_to_python,
-    get_embeded_stream_url,
 )
 
 # engine metadata
@@ -67,7 +68,7 @@ naver_category_dict = {
 }
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if naver_category not in ('general', 'images', 'news', 'videos'):
         raise SearxEngineAPIException(f"Unsupported category: {naver_category}")
 
@@ -193,7 +194,7 @@ def parse_news(data):
 
 
 def parse_videos(data):
-    results = []
+    res = EngineResults()
 
     dom = html.fromstring(data)
 
@@ -212,15 +213,14 @@ def parse_videos(data):
         except (ValueError, TypeError):
             pass
 
-        results.append(
-            {
-                "template": "videos.html",
-                "title": extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
-                "url": url,
-                "thumbnail": thumbnail,
-                "length": length,
-                "iframe_src": get_embeded_stream_url(url),
-            }
+        res.add(
+            res.types.LegacyResult(
+                template="videos.html",
+                title=extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
+                url=url,
+                thumbnail=thumbnail,
+                length=length,
+            )
         )
 
-    return results
+    return res

searx/engines/neosearch.py

@@ -6,10 +6,14 @@
 
 from json import loads
 import typing as t
-from urllib.parse import urlencode
 
+from lxml import html
+
+from searx.exceptions import SearxEngineAPIException
 from searx.extended_types import SXNG_Response
+from searx.network import get
 from searx.result_types import EngineResults
+from searx.utils import eval_xpath, extract_text
 
 if t.TYPE_CHECKING:
     from searx.enginelib.traits import EngineTraits
@@ -25,18 +29,33 @@ about = {
     "results": "JSON",
 }
 
+paging = False
+enable_http3 = True
+
 base_url = "https://neosearch.org"
 categories = ["general"]
 
-paging = False
+
+def _obtain_xsrf_token() -> str:
+    resp = get(base_url)
+    doc = html.fromstring(resp.text)
+
+    xsrf_token = extract_text(eval_xpath(doc, "//meta[@name='xsrf-token']/@content"))
+    if not xsrf_token:
+        raise SearxEngineAPIException("failed to obtain xsrf token")
+    return xsrf_token
 
 
 def request(query: str, params: "OnlineParams"):
+    params["url"] = f"{base_url}/search"
+    params["headers"]["X-XSRF-TOKEN"] = _obtain_xsrf_token()
+    params["method"] = "POST"
+
     args = {"q": query, "generate": "auto"}
     countrycode = params["searxng_locale"].split("-")[-1].upper()
     if countrycode in traits.custom["countrycodes"]:
         args["loc"] = countrycode
-    params["url"] = f"{base_url}/search?{urlencode(args)}"
+    params["json"] = args
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
@@ -67,7 +86,6 @@ def response(resp: "SXNG_Response") -> EngineResults:
 
 def fetch_traits(engine_traits: "EngineTraits") -> None:
     # pylint: disable=import-outside-toplevel
-    from searx.network import get
     from searx.utils import extr, js_obj_str_to_python
     from babel.core import get_global
 

searx/engines/openstreetmap.py

@@ -10,7 +10,8 @@ from flask_babel import gettext
 
 from searx.data import OSM_KEYS_TAGS, CURRENCIES
 from searx.external_urls import get_external_url
-from searx.engines.wikidata import send_wikidata_query, sparql_string_escape, get_thumbnail
+from searx.wikidata import send_wikidata_query
+from searx.engines.wikidata import sparql_string_escape, get_thumbnail
 from searx.result_types import EngineResults
 
 # about
@@ -290,7 +291,8 @@ def get_title_address(result):
                 'house_number': address_raw.get('house_number'),
                 'road': address_raw.get('road'),
                 'locality': address_raw.get(
-                    'city', address_raw.get('town', address_raw.get('village'))  # noqa
+                    'city',
+                    address_raw.get('town', address_raw.get('village')),  # noqa
                 ),  # noqa
                 'postcode': address_raw.get('postcode'),
                 'country': address_raw.get('country'),

searx/engines/pexels.py

@@ -2,12 +2,13 @@
 """Pexels (images)"""
 
 import re
+import typing as t
 
 from urllib.parse import urlencode
 from lxml import html
 
 from searx.result_types import EngineResults
-from searx.utils import eval_xpath_list, gen_useragent
+from searx.utils import eval_xpath_list
 from searx.enginelib import EngineCache
 from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
 from searx.network import get
@@ -43,24 +44,17 @@ SECRET_KEY_DB_KEY = "secret-key"
 CACHE: EngineCache
 """Cache to store the secret API key for the engine."""
 
-enable_http2 = False
 
-
-def init(engine_settings):
+def setup(engine_settings: dict[str, t.Any]) -> bool:
     global CACHE  # pylint: disable=global-statement
     CACHE = EngineCache(engine_settings["name"])
+    return True
 
 
 def _get_secret_key():
     resp = get(
         base_url,
-        headers={
-            # circumvents Cloudflare bot protections
-            "User-Agent": gen_useragent(),
-            "Referer": base_url,
-            "Sec-GPC": "1",
-            "Connection": "keep-alive",
-        },
+        headers={"Referer": base_url},
     )
 
     if resp.status_code != 200:
@@ -103,8 +97,6 @@ def request(query, params):
 
     params["headers"]["secret-key"] = secret_key
 
-    return params
-
 
 def response(resp):
     res = EngineResults()

searx/engines/pinterest.py

@@ -2,72 +2,85 @@
 """Pinterest (images)"""
 
 from json import dumps
+import typing as t
+from searx.result_types import EngineResults
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
 
 about = {
-    "website": 'https://www.pinterest.com/',
-    "wikidata_id": 'Q255381',
-    "official_api_documentation": 'https://developers.pinterest.com/docs/api/v5/',
+    "website": "https://www.pinterest.com/",
+    "wikidata_id": "Q255381",
+    "official_api_documentation": "https://developers.pinterest.com/docs/api/v5/",
     "use_official_api": False,
     "require_api_key": False,
-    "results": 'JSON',
+    "results": "JSON",
 }
 
-categories = ['images']
+categories = ["images"]
 paging = True
 
-base_url = 'https://www.pinterest.com'
+base_url = "https://www.pinterest.com"
 
 
-def request(query, params):
+def request(query: str, params: "OnlineParams") -> None:
+
     args = {
-        'options': {
-            'query': query,
-            'bookmarks': [params['engine_data'].get('bookmark', '')],
+        "options": {
+            "query": query,
+            "bookmarks": [params["engine_data"].get("bookmark", "")],
         },
-        'context': {},
+        "context": {},
     }
-    params['url'] = f"{base_url}/resource/BaseSearchResource/get/?data={dumps(args)}"
-    params['headers'] = {
-        'X-Pinterest-AppState': 'active',
-        'X-Pinterest-Source-Url': '/ideas/',
-        'X-Pinterest-PWS-Handler': 'www/ideas.js',
+    params["url"] = f"{base_url}/resource/BaseSearchResource/get/?data={dumps(args)}"
+    params["headers"] = {
+        "X-Requested-With": "XMLHttpRequest",
+        "X-Pinterest-AppState": "active",
+        "X-Pinterest-Source-Url": "/ideas/",
+        "X-Pinterest-PWS-Handler": "www/ideas.js",
     }
 
-    return params
 
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
+    json_resp: dict[str, t.Any] = resp.json()  # type: ignore
 
-def response(resp):
-    results = []
-
-    json_resp = resp.json()
-
-    results.append(
+    res.add(
         {
-            'engine_data': json_resp['resource_response']['bookmark'],
+            "engine_data": json_resp["resource_response"]["bookmark"],
             # it's called bookmark by pinterest, but it's rather a nextpage
             # parameter to get the next results
-            'key': 'bookmark',
+            "key": "bookmark",
         }
     )
 
-    for result in json_resp['resource_response']['data']['results']:
+    for result in json_resp["resource_response"]["data"]["results"]:
 
-        if result['type'] == 'story':
+        if result["type"] == "story":
             continue
 
-        main_image = result['images']['orig']
-        results.append(
-            {
-                'template': 'images.html',
-                'url': result.get('link') or f"{base_url}/pin/{result['id']}/",
-                'title': result.get('title') or result.get('grid_title'),
-                'content': (result.get('rich_summary') or {}).get('display_description') or "",
-                'img_src': main_image['url'],
-                'thumbnail_src': result['images']['236x']['url'],
-                'source': (result.get('rich_summary') or {}).get('site_name'),
-                'resolution': f"{main_image['width']}x{main_image['height']}",
-                'author': f"{result['pinner'].get('full_name')} ({result['pinner']['username']})",
-            }
+        main_image = result["images"]["orig"]
+
+        title = result.get("title") or result.get("grid_title") or ""
+        if len(title) < 5:
+            visual_annotation = result.get("pin_join", {}).get("visual_annotation")
+            if visual_annotation:
+                title = visual_annotation[0]
+            else:
+                title = result.get("name") or result.get("auto_alt_text") or ""
+
+        res.add(
+            res.types.Image(
+                url=result.get("link") or f"{base_url}/pin/{result['id']}/",
+                title=title,
+                content=(result.get("rich_summary") or {}).get("display_description") or "",
+                img_src=main_image["url"],
+                thumbnail_src=result["images"]["236x"]["url"],
+                source=(result.get("rich_summary") or {}).get("site_name") or "",
+                resolution=f"{main_image['width']}x{main_image['height']}",
+                author=f"{result['pinner'].get('full_name')} ({result['pinner']['username']})",
+            )
         )
 
-    return results
+    return res

searx/engines/pixabay.py

@@ -27,9 +27,6 @@ time_range_support = True
 safesearch_map = {0: 'off', 1: '1', 2: '1'}
 time_range_map = {'day': '1d', 'week': '1w', 'month': '1m', 'year': '1y'}
 
-# using http2 returns forbidden errors
-enable_http2 = False
-
 
 def request(query, params):
     args = {
@@ -50,8 +47,6 @@ def request(query, params):
     # prevent automatic redirects to first page on pagination
     params['allow_redirects'] = False
 
-    return params
-
 
 def _image_result(result):
     return {

searx/engines/postgresql.py

@@ -21,6 +21,8 @@ Implementations
 
 """
 
+import typing as t
+
 try:
     import psycopg2  # type: ignore
 except ImportError:
@@ -55,15 +57,17 @@ paging = True
 _connection = None
 
 
-def init(engine_settings):
-    global _connection  # pylint: disable=global-statement
-
+def setup(engine_settings: dict[str, t.Any]) -> bool | None:
     if 'query_str' not in engine_settings:
         raise ValueError('query_str cannot be empty')
 
     if not engine_settings['query_str'].lower().startswith('select '):
         raise ValueError('only SELECT query is supported')
 
+
+def init(_):
+    global _connection  # pylint: disable=global-statement
+
     _connection = psycopg2.connect(
         database=database,
         user=username,

searx/engines/privacywall.py

@@ -14,7 +14,7 @@ from lxml import html
 import babel
 
 from searx.enginelib.traits import EngineTraits
-from searx.utils import eval_xpath_list, eval_xpath, extract_text, get_embeded_stream_url, extr
+from searx.utils import eval_xpath_list, eval_xpath, extract_text, extr
 from searx.locales import region_tag
 from searx.result_types import EngineResults
 
@@ -61,7 +61,7 @@ video_page_map = {
 }
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if privacywall_category not in ("general", "images", "videos"):
         raise ValueError("invalid category: %s" % privacywall_category)
 
@@ -154,7 +154,6 @@ def _video_results(doc: "ElementBase") -> EngineResults:
                 title=extract_text(eval_xpath(result, ".//h2[contains(@class, 'video-card-title')]")) or "",
                 content=extract_text(eval_xpath(result, ".//p")) or "",
                 thumbnail=thumbnail or "",
-                iframe_src=get_embeded_stream_url(url) or "",
             )
         )
 

searx/engines/quark.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Quark (Shenma) search engine for searxng"""
 
+import typing as t
 from urllib.parse import urlencode
 from datetime import datetime
 import re
@@ -43,7 +44,7 @@ def is_alibaba_captcha(html):
     return bool(re.search(CAPTCHA_PATTERN, html))
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if quark_category not in ('general', 'images'):
         raise SearxEngineAPIException(f"Unsupported category: {quark_category}")
 

searx/engines/qwant.py

@@ -60,9 +60,6 @@ from searx.exceptions import (
     SearxEngineTooManyRequestsException,
 )
 from searx.network import raise_for_httperror
-from searx.utils import (
-    get_embeded_stream_url,
-)
 from searx.result_types import EngineResults
 
 if t.TYPE_CHECKING:
@@ -299,7 +296,6 @@ def response(resp: "SXNG_Response") -> EngineResults:
                         title=title,
                         url=res_url,
                         content=content,
-                        iframe_src=get_embeded_stream_url(res_url),
                         publishedDate=pub_date,
                         thumbnail=thumbnail,
                         template="videos.html",

searx/engines/resulthunter.py

@@ -10,6 +10,7 @@ from urllib.parse import urlencode
 from lxml import html
 
 from searx import locales
+from searx.exceptions import SearxEngineResponseException
 from searx.result_types import EngineResults
 from searx.utils import eval_xpath_list, eval_xpath, extract_text
 
@@ -42,7 +43,7 @@ resulthunter_categ = "web"
 """Supported categories are ``web`` and ``images``."""
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if resulthunter_categ not in ("web", "images"):
         raise ValueError("invalid category: %s" % resulthunter_categ)
 
@@ -52,6 +53,7 @@ def request(query: str, params: "OnlineParams") -> None:
         "q": query,
         "search_type": resulthunter_categ,
         "offset": params["pageno"] - 1,
+        "search_source": "other",
     }
 
     # uses Brave's engine traits
@@ -111,6 +113,11 @@ def _image_results(doc: "ElementBase") -> EngineResults:
 def response(resp: "SXNG_Response") -> EngineResults:
     doc = html.fromstring(resp.text)
 
+    # if the request was wrong (e.g. missing params), the site doesn't contain a result container
+    # and instead shows an "Installation required" page to download the resulthunter browser extension
+    if not eval_xpath(doc, "//div[contains(@class, 'organic-results-container')]"):
+        raise SearxEngineResponseException()
+
     match resulthunter_categ:
         case "web":
             return _general_results(doc)

searx/engines/s1search.py

@@ -38,12 +38,10 @@ CACHE: EngineCache
 """Cache to store verification tokens for pagination."""
 
 
-def init(_):
+def setup(engine_settings: dict[str, t.Any]) -> bool:
     if not base_url:
         raise ValueError("base_url must be set")
 
-
-def setup(engine_settings: dict[str, t.Any]) -> bool:
     global CACHE  # pylint: disable=global-statement
     CACHE = EngineCache(engine_settings["name"])
     return True
@@ -82,7 +80,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
             res.types.MainResult(
                 url=extract_text(eval_xpath(result, ".//a[contains(@class, 'title')]/@href")),
                 title=extract_text(eval_xpath(result, ".//a[contains(@class, 'title')]")),
-                content=extract_text(eval_xpath(result, ".//span[contains(@class, 'description') or @class='']")),
+                content=extract_text(eval_xpath(result, ".//span[contains(@class, 'description') or not(@class)]")),
             )
         )
 

searx/engines/s1search_rampjs.py

@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""JavaScript-based s1search implementation. See :ref:`s1search engine`.
+
+Works for all s1search sites that contain the ``__RAMPJS__`` JavaScript variable.
+"""
+
+import json
+import typing as t
+from urllib.parse import urlencode
+
+from searx.result_types import EngineResults
+from searx.utils import extr, html_to_text
+
+if t.TYPE_CHECKING:
+    from searx.search.processors import OnlineParams
+    from searx.extended_types import SXNG_Response
+
+about = {
+    "website": "https://s1search.co",
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "JSON",
+}
+
+categories = ["general"]
+paging = True
+
+base_url = "https://search.answers.com"
+# other working base URLs:
+# - https://search.nation.online
+# - https://search.activebeat.com
+# - https://search.legalboulevard.com
+# - https://search.walletgenius.com
+# - https://search.legalboulevard.com
+
+
+def request(query: str, params: "OnlineParams"):
+    args = {"q": query, "page": params["pageno"]}
+    params["url"] = f"{base_url}/?{urlencode(args)}"
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
+
+    data_raw = extr(resp.text, "response: ", " };")
+    data = json.loads(data_raw)
+
+    mainline = [s for s in data["search"]["regions"] if s["name"] == "mainline"][0]
+    for group in mainline["groups"]:
+        for result in group["results"]:
+            if not ("url" in result or "clickUrl" in result):
+                continue
+
+            res.add(
+                res.types.MainResult(
+                    url=result.get("url") or result.get("clickUrl"),
+                    title=html_to_text(result["title"]),
+                    content=html_to_text(result["description"]),
+                )
+            )
+
+    return res

searx/engines/searchrockit.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""SearchRockit is an American search engine. It allegedly has its own index,
+but the results seem to come from Google."""
+
+import typing as t
+from urllib.parse import urlencode
+from lxml import html
+from dateutil import parser
+
+from searx.result_types import EngineResults
+from searx.utils import (
+    eval_xpath_list,
+    extract_text,
+    eval_xpath,
+)
+
+if t.TYPE_CHECKING:
+    from searx.search.processors import OnlineParams
+    from searx.extended_types import SXNG_Response
+
+about = {
+    "website": "https://searchrockit.com",
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+}
+
+categories = ["general"]
+paging = True
+
+SearchrockitCateg = t.Literal["web", "images", "news"]
+searchrockit_categ: SearchrockitCateg = "web"
+
+base_url = "https://searchrockit.com"
+
+
+def setup(_):
+    if searchrockit_categ not in t.get_args(SearchrockitCateg):
+        raise ValueError("invalid search category: %s" % searchrockit_categ)
+
+
+def request(query: str, params: "OnlineParams") -> None:
+    args = {"q": query, "p": params["pageno"]}
+    params["url"] = f"{base_url}/results/{searchrockit_categ}?{urlencode(args)}"
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    doc = html.fromstring(resp.text)
+    res = EngineResults()
+
+    match searchrockit_categ:
+        case "web" | "news":
+            for result in eval_xpath_list(
+                doc, "//div[contains(@class, 'results-list')]/div[contains(@class, 'result-item')]"
+            ):
+                publishedDate = None
+                try:
+                    d = extract_text(eval_xpath(result, ".//span[contains(@class, 'result-item--publishedAt')]")) or ""
+                    publishedDate = parser.parse(d)
+                except parser.ParserError:
+                    pass
+                res.add(
+                    res.types.MainResult(
+                        url=extract_text(eval_xpath(result, ".//a[contains(@class, 'result-item--title')]/@href")),
+                        title=extract_text(eval_xpath(result, ".//a[contains(@class, 'result-item--title')]")) or "",
+                        content=extract_text(eval_xpath(result, ".//a[contains(@class, 'result-item--desc')]")) or "",
+                        thumbnail=extract_text(
+                            eval_xpath(result, ".//a[contains(@class, 'result-item--thumb')]/img/@src")
+                        )
+                        or "",
+                        publishedDate=publishedDate,
+                    )
+                )
+        case "images":
+            for result in eval_xpath_list(
+                doc, "//div[contains(@class, 'image-grid')]/a[contains(@class, 'image-card')]"
+            ):
+                res.add(
+                    res.types.Image(
+                        url=extract_text(eval_xpath(result, "./@href")),
+                        title=extract_text(eval_xpath(result, "./div[contains(@class, 'image-title')]")) or "",
+                        thumbnail_src=extract_text(eval_xpath(result, "./img/@src")) or "",
+                        img_src=extract_text(eval_xpath(result, "./@data-full-url")) or "",
+                    )
+                )
+
+    return res

searx/engines/searchzee.py

@@ -4,13 +4,11 @@ independent search infrastructure."""
 
 import typing as t
 from urllib.parse import urlencode
+import uuid
 
-from searx.exceptions import SearxEngineAPIException
 from searx.extended_types import SXNG_Response
-from searx.network import get
 from searx.result_types import EngineResults
-from searx.utils import extr, html_to_text
-from searx.enginelib import EngineCache
+from searx.utils import html_to_text
 
 if t.TYPE_CHECKING:
     from searx.search.processors import OnlineParams
@@ -34,43 +32,19 @@ SearchzeeCategType = t.Literal["web", "news"]
 searchzee_categ: SearchzeeCategType = None  # type: ignore[reportAssignmentType]
 
 
-CACHE: EngineCache
-"""Cache for storing the scraped API Token."""
-
 base_url = "https://searchzee.com"
 
 # only supports for news
 time_range_map = {"day": "pd", "week": "pw", "month": "pm", "year": "py"}
 
 
-def setup(engine_settings: dict[str, t.Any]) -> bool:
+def setup(_: dict[str, t.Any]):
     if searchzee_categ not in t.get_args(SearchzeeCategType):
         raise ValueError("invalid category: %s" % searchzee_categ)
 
-    global CACHE  # pylint: disable=global-statement
-    CACHE = EngineCache(engine_settings["name"])  # type: ignore[reportAny]
-    return True
-
-
-def _obtain_api_token() -> str:
-    token: str | None = CACHE.get("token")  # type: ignore[reportAny]
-    if token:
-        return token
-
-    token_resp = get(
-        f"{base_url}/app.js",
-    )
-    if not token_resp.ok:
-        raise SearxEngineAPIException("failed to obtain api key")
-
-    token = extr(token_resp.text, "const SEARCHZEE_API_TOKEN = \"", "\";")
-    CACHE.set("token", token, expire=3600)
-
-    return token
-
 
 def request(query: str, params: "OnlineParams"):
-    params["headers"]["X-SearchZee-Token"] = _obtain_api_token()
+    params["cookies"]["szs"] = str(uuid.uuid4())
 
     args = {"q": query, "type": searchzee_categ, "offset": params["pageno"] - 1}
     if params["time_range"]:

searx/engines/searx_engine.py

@@ -1,57 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""
-Searx (all)
-"""
-
-from json import loads
-from searx.engines import categories as searx_categories
-
-# about
-about = {
-    "website": 'https://github.com/searxng/searxng',
-    "wikidata_id": 'Q17639196',
-    "official_api_documentation": 'https://docs.searxng.org/dev/search_api.html',
-    "use_official_api": True,
-    "require_api_key": False,
-    "results": 'JSON',
-}
-
-categories = searx_categories.keys()
-
-# search-url
-instance_urls = []
-instance_index = 0
-
-
-# do search-request
-def request(query, params):
-    global instance_index  # pylint: disable=global-statement
-    params['url'] = instance_urls[instance_index % len(instance_urls)]
-    params['method'] = 'POST'
-
-    instance_index += 1
-
-    params['data'] = {
-        'q': query,
-        'pageno': params['pageno'],
-        'language': params['language'],
-        'time_range': params['time_range'],
-        'category': params['category'],
-        'format': 'json',
-    }
-
-    return params
-
-
-# get response from search-request
-def response(resp):
-
-    response_json = loads(resp.text)
-    results = response_json['results']
-
-    for i in ('answers', 'infoboxes'):
-        results.extend(response_json[i])
-
-    results.extend({'suggestion': s} for s in response_json['suggestions'])
-
-    return results

searx/engines/solr.py

@@ -28,7 +28,7 @@ This is an example configuration for searching in the collection
 """
 
 # pylint: disable=global-statement
-
+import typing as t
 from urllib.parse import urlencode
 from searx.exceptions import SearxEngineAPIException
 from searx.result_types import EngineResults
@@ -46,7 +46,7 @@ _search_url = ''
 paging = True
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if collection == '':
         raise ValueError('collection cannot be empty')
 

searx/engines/soundcloud.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """SoundCloud is a German audio streaming service."""
 
+import typing as t
 import re
 import datetime
 
@@ -115,9 +116,10 @@ def response(resp):
     return results
 
 
-def init(engine_settings):  # pylint: disable=unused-argument
+def setup(engine_settings: dict[str, t.Any]) -> bool:
     global CACHE  # pylint: disable=global-statement
     CACHE = EngineCache(engine_settings["name"])  # type:ignore
+    return True
 
 
 def get_client_id() -> str | None:

searx/engines/sqlite.py

@@ -66,7 +66,7 @@ limit = 10
 paging = True
 
 
-def init(engine_settings):
+def setup(engine_settings: dict[str, t.Any]) -> bool | None:
     if 'query_str' not in engine_settings:
         raise ValueError('query_str cannot be empty')
 

searx/engines/startpage.py

@@ -1,5 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Startpage's language & region selectors are a mess ..
+"""Startpage requires solving an Anubis POW captcha (difficulty 4).
+Solving it requires a lot of CPU, so the engine is set inactive by default.
+
+Startpage's language & region selectors are a mess ..
 
 .. _startpage regions:
 
@@ -84,6 +87,7 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
 """
 # pylint: disable=too-many-statements
 
+import hashlib
 import re
 import typing as t
 from collections import OrderedDict
@@ -99,7 +103,7 @@ from searx.enginelib import EngineCache
 from searx.enginelib.traits import EngineTraits
 from searx.exceptions import SearxEngineCaptchaException
 from searx.locales import region_tag
-from searx.network import get  # see https://github.com/searxng/searxng/issues/762
+from searx.network import get, post  # see https://github.com/searxng/searxng/issues/762
 from searx.utils import (
     eval_xpath,
     extr,
@@ -165,7 +169,7 @@ CACHE: EngineCache
 seconds."""
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     global CACHE  # pylint: disable=global-statement
 
     # hint: all three startpage engines (WEB, Images & News) can/should use the
@@ -176,6 +180,45 @@ def init(_):
 sc_code_cache_sec = 3600
 """Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
 
+# startpage's anubis difficulty is set to 4
+max_difficulty = 4
+
+
+def _solve_anubis(resp) -> str:
+    """Anubis POW solver"""
+    payload = loads(extr(resp.text, '<script id="anubis_challenge" type="application/json">', "</script>"))
+    challenge = payload["challenge"]
+    difficulty = int(payload["rules"]["difficulty"])
+    if difficulty > max_difficulty:
+        raise SearxEngineCaptchaException(message="startpage: Anubis difficulty too high")
+    prefix = "0" * difficulty
+    blob = challenge["randomData"].encode()
+    for nonce in range(16**difficulty * 8):  # max search is 8x average search, e^-8 = 0.034% will fail
+        digest = hashlib.sha256(blob + str(nonce).encode()).hexdigest()
+        if digest.startswith(prefix):
+            break
+    else:
+        raise SearxEngineCaptchaException(message="startpage: Anubis failed")
+
+    pass_resp = get(
+        f"{base_url}/.within.website/x/cmd/anubis/api/pass-challenge",
+        params={
+            "id": challenge["id"],
+            "response": digest,
+            "nonce": nonce,
+            "redir": str(resp.url),
+            "elapsedTime": "1",
+        },
+        cookies=resp.cookies,
+        allow_redirects=False,
+    )
+    auth = pass_resp.cookies.get("spchal-auth")
+    if not auth:
+        raise SearxEngineCaptchaException(message="startpage: Anubis pass-challenge failed")
+    auth = str(auth)
+    CACHE.set("SPCHAL_AUTH", auth, expire=240)
+    return auth
+
 
 def get_sc_code(params):
     """Get an actual ``sc`` argument from Startpage's search form (HTML page).
@@ -201,6 +244,9 @@ def get_sc_code(params):
     logger.debug("get_sc_code: request headers: %s", headers)
     resp = get(get_sc_url, headers=headers)
 
+    if 'id="anubis_challenge"' in resp.text:
+        resp = get(get_sc_url, headers=headers, cookies={"spchal-auth": _solve_anubis(resp)})
+
     # ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)
     # ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg
     # ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21
@@ -239,8 +285,8 @@ def request(query, params):
     Additionally the arguments form Startpage's search form needs to be set in
     HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.
     """
-    engine_region = traits.get_region(params["searxng_locale"], "en-US")
-    engine_language = traits.get_language(params["searxng_locale"], "en")
+    engine_region = traits.get_region(params["searxng_locale"], "en_US")
+    engine_language = traits.get_language(params["searxng_locale"], "english")
 
     params["headers"]["Origin"] = base_url
     params["headers"]["Referer"] = base_url + "/"
@@ -262,9 +308,9 @@ def request(query, params):
         args["language"] = engine_language
         args["lui"] = engine_language
 
+    args["segment"] = "startpage.udog"
     if params["pageno"] > 1:
         args["page"] = params["pageno"]
-        args["segment"] = "startpage.udog"
 
     # Build cookie
     lang_homepage = "en"
@@ -289,6 +335,8 @@ def request(query, params):
         cookie["search_results_region"] = engine_region
 
     params["cookies"]["preferences"] = "N1N".join(["%sEEE%s" % x for x in cookie.items()])
+    if auth := CACHE.get("SPCHAL_AUTH"):
+        params["cookies"]["spchal-auth"] = auth
     logger.debug("cookie preferences: %s", params["cookies"]["preferences"])
 
     logger.debug("data: %s", args)
@@ -400,6 +448,18 @@ def _get_image_result(result) -> dict[str, t.Any] | None:
 
 
 def response(resp):
+    if 'id="anubis_challenge"' in resp.text:
+        params = resp.search_params
+        params["cookies"]["spchal-auth"] = _solve_anubis(resp)
+        resp = post(
+            params["url"] or search_url,
+            data=params["data"],
+            headers=params["headers"],
+            cookies=params["cookies"],
+        )
+        if 'id="anubis_challenge"' in resp.text:
+            raise SearxEngineCaptchaException()
+
     categ = startpage_categ.capitalize()
     results_raw = "{" + extr(resp.text, f"React.createElement(UIStartpage.AppSerp{categ}, {{", "}})") + "}}"
 

searx/engines/startpagina.py

@@ -39,7 +39,7 @@ page_size = 10
 api_url = "https://search.kompas.services"
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if startpagina_categ not in ("web", "images", "videos", "news"):
         raise ValueError("invalid search type: %s" % startpagina_categ)
 

searx/engines/stocksnap.py

@@ -21,8 +21,6 @@ about = {
     "require_api_key": False,
     "results": "JSON",
 }
-# otherwise all requests get blocked, probably HTTP2 fingerprinting
-enable_http2 = False
 
 base_url = "https://stocksnap.io"
 cdn_url = "https://cdn.stocksnap.io"

searx/engines/swisscows.py

@@ -152,7 +152,7 @@ def generate_nonce_and_signature(base_path: str, args: dict[str, t.Any]) -> tupl
 maximum_page_size = {"web": 20, "images": 50, "videos": 10}
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if swisscows_category not in ("web", "images", "videos"):
         raise ValueError("illegal swisscows category: %s" % swisscows_category)
 

searx/engines/tiger.py

@@ -45,12 +45,10 @@ CACHE: EngineCache
 """Cache to store session codes (result of solved CAPTCHA)."""
 
 
-def init(_):
+def setup(engine_settings: dict[str, t.Any]) -> bool:
     if tiger_category not in ("Websuche", "News"):
         raise ValueError("invalid search category: %s" % tiger_category)
 
-
-def setup(engine_settings: dict[str, t.Any]) -> bool:
     global CACHE  # pylint: disable=global-statement
     CACHE = EngineCache(engine_settings["name"])
     return True

searx/engines/tonline.py

@@ -14,7 +14,7 @@ from urllib.parse import urlencode
 
 from lxml import html
 
-from searx.utils import eval_xpath_list, eval_xpath, extract_text, get_embeded_stream_url, ElementType
+from searx.utils import eval_xpath_list, eval_xpath, extract_text, ElementType
 from searx.result_types import EngineResults
 from searx.enginelib import EngineAbout
 
@@ -44,7 +44,7 @@ tonline_channel_map = {"images": "flickr", "videos": "yt"}
 language = "de"
 
 
-def init(_):
+def setup(_: dict[str, t.Any]) -> bool | None:
     if tonline_categ not in ("web", "images", "videos", "news"):
         raise ValueError("invalid category: %s" % tonline_categ)
 
@@ -126,7 +126,6 @@ def _video_results(doc: ElementType, res: EngineResults):
                 url=url,
                 title=" - ".join(extract_text(part) or "" for part in title_parts),
                 thumbnail=extract_text(eval_xpath(result, ".//img/@src") or "") or "",
-                iframe_src=get_embeded_stream_url(url) or "",
             )
         )
 

searx/engines/torznab.py

@@ -83,7 +83,7 @@ show_torrent_files: bool = False
 show_magnet_links: bool = True
 
 
-def init(engine_settings=None):  # pylint: disable=unused-argument
+def setup(_: dict[str, t.Any]) -> bool | None:
     """Initialize the engine."""
     if len(base_url) < 1:
         raise ValueError('missing torznab base_url')