[fix] autocomplete suggestrelevance scores for chromium

Qwant now requires a `datadome` cookie that it returns
in the first search response as a `Set-Cookie`.

This cookie has to be sent for all requests, otherwise they
will be blocked.

This means that now, the first search request is blocked (results in CAPTCHA),
and only the subsequent searches work (same happens on the Qwant website for me).

However, I don't think it's worth repeating the same
search request multiple times very quickly because
that also makes us more suspicious.

searx/engines/qwant.py

@@ -51,6 +51,7 @@ from urllib.parse import urlencode
 import babel
 from flask_babel import gettext  # pyright: ignore[reportUnknownVariableType]
 
+from searx.enginelib import EngineCache
 from searx.enginelib.traits import EngineTraits
 from searx.exceptions import (
     SearxEngineAccessDeniedException,
@@ -105,9 +106,19 @@ qwant_news_locales = [
 ]
 # fmt: on
 
+base_url = "https://www.qwant.com"
 api_url = "https://api.qwant.com/v3/search/"
 """URL of Qwant's API (JSON)"""
 
+CACHE: EngineCache
+"""Cache for storing the ``datadome`` cookie."""
+
+
+def setup(engine_settings: dict[str, t.Any]) -> bool:
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])
+    return True
+
 
 def request(query: str, params: "OnlineParams") -> None:
     """Qwant search request"""
@@ -129,24 +140,29 @@ def request(query: str, params: "OnlineParams") -> None:
         "tgp": test_group_value,
         "device": "desktop",
         "safesearch": params["safesearch"],
-        "display": True,
-        "llm": True,
+        # True would be encoded to "True", instead of "true", which makes the request
+        # easier to detect and block
+        "displayed": "true",
+        "llm": "true",
     }
 
-    # shuffle query parameters to be harder to fingerprint
-    args = list(args.items())
-    random.shuffle(args)
-    args = dict(args)
-
     params["raise_for_httperror"] = False
 
     params["url"] = f"{api_url}{qwant_categ}?{urlencode(args)}"
 
+    params["cookies"]["datadome"] = CACHE.get("datadome")
+    params["headers"].update({"Accept": "application/json", "Referer": f"{base_url}/", "Origin": base_url})
+
 
 def response(resp: "SXNG_Response") -> EngineResults:
     """Parse results from Qwant's API"""
     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 
+    # cache datadome cookie - changes on each request
+    datadome = resp.cookies.get("datadome")
+    if datadome:
+        CACHE.set("datadome", datadome)
+
     res = EngineResults()
 
     # Try to load JSON result
@@ -163,8 +179,8 @@ def response(resp: "SXNG_Response") -> EngineResults:
         error_code = data.get("error_code")
         if error_code == 24:
             raise SearxEngineTooManyRequestsException()
-        if search_results.get("data", {}).get("error_data", {}).get("captchaUrl") is not None:
-            raise SearxEngineCaptchaException()
+        if search_results.get("url") is not None:
+            raise SearxEngineCaptchaException(suspended_time=0)
         if resp.status_code == 403:
             raise SearxEngineAccessDeniedException()
         msg = ",".join(data.get("message", ["unknown"]))

searx/result_types/_base.py

@@ -52,6 +52,12 @@ def _normalize_url_fields(result: "Result | LegacyResult"):
             result.parsed_url = urllib.parse.urlparse(result.url)
 
     if result.parsed_url:
+        # properly format special characters (e.g. "ä", "ö") in IDN domains
+        # e.g. "xn--strung-xxa.de" becomes "störung.de"
+        if result.parsed_url.netloc.startswith("xn--"):
+            netloc = result.parsed_url.netloc.encode().decode("idna")
+            result.parsed_url = result.parsed_url._replace(netloc=netloc)
+
         result.parsed_url = result.parsed_url._replace(
             # if the result has no scheme, use http as default
             scheme=result.parsed_url.scheme or "http",

searx/settings.yml

@@ -477,6 +477,24 @@ engines:
     engine: arxiv
     shortcut: arx
 
+  - name: avalw
+    engine: json_engine
+    shortcut: av
+    categories: general
+    paging: true
+    search_url: https://avalw.org/api/search?q={query}&page={pageno}&limit=10&tbm=all
+    results_query: results
+    url_query: url
+    title_query: title
+    content_query: snippet
+    thumbnail: image_url
+    disabled: true
+    inactive: true
+    about:
+      website: https://avalw.com
+      description: "Search engine from the Romanian news company AVALW S.R.L."
+      results: JSON
+
   - name: ayo
     engine: xpath
     categories: general

searx/templates/simple/result_templates/images.html

@@ -60,7 +60,7 @@
       </p>
       <p class="result-filesize">{{ _label(_("Filesize"), result.filesize) }}</p>
       <p class="result-source">{{ _label(_("Source"), result.source) }}</p>
-      <p class="result-engine">{{ _label(_("Engine"), result.engine) }}</p>
+      <p class="result-engines">{{ _label(_("Engines"), ", ".join(result.engines)) }}</p>
       <p class="result-url"><span>{{ _("View source") }}:</span>{{- "" -}}
         <a {{ _target(results_on_new_tab) }} href="{{ result.url }}">{{ result.url }}</a>
       </p>

searx/webapp.py

@@ -845,7 +845,10 @@ def autocompleter():
         mimetype = 'application/json'
     else:
         # the suggestion request comes from browser's URL bar
-        suggestions = json.dumps([sug_prefix, results])
+        relevances = {
+            'google:suggestrelevance': [600 - i for i in range(len(results))]
+        }  # chromium only shows 3 suggestions unless we attach relevances
+        suggestions = json.dumps([sug_prefix, results, [], [], relevances])
         mimetype = 'application/x-suggestions+json'
 
     suggestions = escape(suggestions, False)