[feat] engines: add paid yandex search api (#6349)

This reverts commit e8e710e42a.

docs/admin/settings/settings_search.rst

@@ -8,7 +8,7 @@
 
    search:
      safe_search: 0
-     autocomplete: ""
+     autocomplete: "duckduckgo"
      favicon_resolver: ""
      default_lang: ""
      ban_time_on_fail: 5
@@ -32,7 +32,7 @@
   - ``2``: Strict
 
 ``autocomplete``:
-  Existing autocomplete backends, leave blank to turn it off.
+  Existing autocomplete backends, set blank to turn it off.
 
   - ``360search``
   - ``baidu``

docs/admin/settings/settings_server.rst

@@ -14,7 +14,7 @@
        limiter: false
        public_instance: false
        image_proxy: false
-       method: "POST"
+       method: "GET"
        default_http_headers:
          X-Content-Type-Options : nosniff
          X-Download-Options : noopen
@@ -58,8 +58,8 @@
 
 ``method`` : ``GET`` | ``POST``
 
-  HTTP method.  By defaults ``POST`` is used / The ``POST`` method has the
+  HTTP method.  By default, ``GET`` is used / The ``POST`` method has the
-  advantage with some WEB browsers that the history is not easy to read, but
+  advantage with some browsers that the history is not saved, but
   there are also various disadvantages that sometimes **severely restrict the
   ease of use for the end user** (e.g. back button to jump back to the previous
   search page and drag & drop of search term to new tabs do not work as

docs/dev/engines/online/yandex_api.rst

@@ -0,0 +1,8 @@
+.. _yandex api engine:
+
+=================
+Yandex Search API
+=================
+
+.. automodule:: searx.engines.yandex_api
+   :members:

searx/engines/dogpile.py

@@ -43,9 +43,10 @@ def init(_):
 
 def request(query: str, params: "OnlineParams"):
     params["url"] = f"{base_url}/api/{dogpile_categ}"
+    params["headers"]["Origin"] = base_url
+
     params["method"] = "POST"
     params["json"] = {"q": query, "qadf": safe_search_map[params["safesearch"]], "page": params["pageno"]}
-    return params
 
 
 def response(resp: "SXNG_Response"):

searx/engines/yandex_api.py

@@ -0,0 +1,217 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Yandex Search API (the official **paid** `Yandex Search API v2`_).
+
+Unlike the :origin:`yandex <searx/engines/yandex.py>` engine (which scrapes the
+public HTML interface and is prone to CAPTCHA blocking), this engine talks to
+the official, paid Yandex Cloud Search API.  It requires a Yandex Cloud account,
+a *folder id* and an *API key*.
+
+The API answers with a Base64-encoded XML document, which is decoded and
+parsed here.
+
+Configuration
+=============
+
+The engine is inactive by default because it needs credentials.  To enable it,
+set ``inactive: false`` and add your ``api_key`` and ``yandex_folder_id`` to
+:origin:`searx/settings.yml`:
+
+.. code:: yaml
+
+  - name: yandex api
+    engine: yandex_api
+    shortcut: yda
+    categories: [general, web]
+    inactive: false
+    api_key: ""           # Yandex Cloud API key (``Api-Key``)
+    yandex_folder_id: ""  # Yandex Cloud folder id
+    # optional, see below:
+    yandex_default_language: en
+
+.. _Yandex Search API v2:
+   https://aistudio.yandex.ru/docs/en/search-api/api-ref/WebSearch/search.html
+"""
+
+import math
+import typing as t
+from base64 import b64decode
+
+from lxml import etree
+
+from searx.exceptions import SearxEngineAPIException
+from searx.result_types import EngineResults
+from searx.utils import extract_text
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
+
+
+about = {
+    "website": "https://yandex.cloud/en/services/search-api",
+    "wikidata_id": "Q5281",
+    "official_api_documentation": "https://aistudio.yandex.ru/docs/en/search-api/api-ref/WebSearch/search.html",
+    "use_official_api": True,
+    "require_api_key": True,
+    "results": "XML",
+}
+
+# Engine configuration
+categories = ["general", "web"]
+paging = True
+safesearch = True
+
+# Credentials, overwritten via settings.yml
+api_key: str = ""
+"""Yandex Cloud API key, passed as ``Authorization: Api-Key <api_key>``."""
+
+yandex_folder_id: str = ""
+"""Yandex Cloud folder id the API key belongs to."""
+
+# Search tuning, overwritten via settings.yml
+yandex_default_language: str = "en"
+"""Default query language.  It selects the Yandex search domain (e.g. yandex.ru
+for ``ru``, yandex.com for ``en``) and the language of the search-result
+notifications, but only as a fallback -- a request whose own locale matches
+:py:obj:`language_map` overrides it.  Must be one of its keys: ``ru``, ``be``,
+``kk``, ``uk``, ``tr`` or ``en``."""
+
+region: str = ""
+"""Optional Yandex `region id`.
+Only meaningful together with ``SEARCH_TYPE_RU``.
+
+__ https://aistudio.yandex.ru/docs/en/search-api/reference/regions.html
+"""
+
+page_size: int = 10
+"""Number of results requested per page."""
+
+base_url = "https://searchapi.api.cloud.yandex.net/v2/web/search"
+
+# searxng safesearch level -> Yandex familyMode
+safesearch_map = {
+    0: "FAMILY_MODE_NONE",
+    1: "FAMILY_MODE_MODERATE",
+    2: "FAMILY_MODE_STRICT",
+}
+
+# Map a query language to a (search_type, l10n) pair.  It drives both the
+# per-request override (when the query's locale matches) and the
+# ``yandex_default_language`` default.
+language_map = {
+    "ru": ("SEARCH_TYPE_RU", "LOCALIZATION_RU"),
+    "be": ("SEARCH_TYPE_BE", "LOCALIZATION_BE"),
+    "kk": ("SEARCH_TYPE_KK", "LOCALIZATION_KK"),
+    "uk": ("SEARCH_TYPE_RU", "LOCALIZATION_UK"),
+    "tr": ("SEARCH_TYPE_TR", "LOCALIZATION_TR"),
+    "en": ("SEARCH_TYPE_COM", "LOCALIZATION_EN"),
+    # Uzbek ('uz') is intentionally omitted: Yandex offers SEARCH_TYPE_UZ but no
+    # matching LOCALIZATION_UZ, so such queries fall back to the defaults above.
+}
+
+
+def setup(_):
+    """Validate credentials and paging limits when the engine is loaded."""
+    if not api_key or not yandex_folder_id:
+        raise SearxEngineAPIException("missing 'api_key' and/or 'yandex_folder_id' in engine settings")
+    if not 1 <= page_size <= 100:
+        raise SearxEngineAPIException("'page_size' must be in the range 1..100 (Yandex 'groupsOnPage')")
+    if yandex_default_language not in language_map:
+        raise SearxEngineAPIException(f"'yandex_default_language' must be one of {sorted(language_map)}")
+
+
+def request(query: str, params: "OnlineParams"):
+
+    # Yandex returns at most 250 results for a query.
+    max_page = math.ceil(250 / page_size)
+    if params["pageno"] > max_page:
+        params["url"] = None
+        return
+
+    if len(query) > 400:
+        # Yandex rejects a 'queryText' longer than 400 characters; decline the
+        # request gracefully instead of provoking an API error.
+        params["url"] = None
+        return
+
+    lang = params["searxng_locale"].split("-")[0].lower()
+    req_search_type, req_l10n = language_map.get(lang, language_map[yandex_default_language])
+
+    body: dict[str, t.Any] = {
+        "query": {
+            "searchType": req_search_type,
+            "queryText": query,
+            "familyMode": safesearch_map[params["safesearch"]],
+            # the API uses a 0-based page index
+            "page": str(params["pageno"] - 1),
+        },
+        "groupSpec": {
+            "groupMode": "GROUP_MODE_FLAT",
+            "groupsOnPage": str(page_size),
+            "docsInGroup": "1",
+        },
+        "l10n": req_l10n,
+        "folderId": yandex_folder_id,
+        "responseFormat": "FORMAT_XML",
+    }
+    # Yandex accepts a 'region' only together with the Russian search type.
+    if region and req_search_type == "SEARCH_TYPE_RU":
+        body["region"] = region
+
+    params["method"] = "POST"
+    params["url"] = base_url
+    params["headers"]["Authorization"] = f"Api-Key {api_key}"
+    params["headers"]["Content-Type"] = "application/json"
+    params["json"] = body
+
+
+def _raw_xml(resp: "SXNG_Response") -> bytes:
+    """Extract and Base64-decode the XML payload out of the JSON envelope.
+
+    The synchronous ``/v2/web/search`` endpoint returns ``{"rawData":
+    "<base64>"}`` on success; HTTP errors are raised upstream via
+    ``raise_for_httperror``.
+    """
+    data: dict[str, t.Any] = resp.json()
+
+    raw_data = data.get("rawData")
+    if raw_data is None:
+        raise SearxEngineAPIException("Yandex Search API: no 'rawData' in response")
+
+    return b64decode(raw_data)
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
+
+    dom = etree.fromstring(_raw_xml(resp))  # pylint: disable=c-extension-no-member
+
+    # An <error> inside <response> signals an application error.  Code 15 simply
+    # means "nothing was found" and must not raise.
+    error = dom.find(".//response/error")
+    if error is not None:
+        if error.get("code") == "15":
+            return res
+        raise SearxEngineAPIException(f"Yandex Search API error {error.get('code')}: {error.text}")
+
+    for doc in dom.iterfind(".//doc"):
+        url = extract_text(doc.find("url"), allow_none=True)
+        title = extract_text(doc.find("title"), allow_none=True)
+        if not url or not title:
+            continue
+
+        content = extract_text(doc.find("headline"), allow_none=True)
+        if not content:
+            passages = doc.find("passages")
+            if passages is not None:
+                content = " ".join(extract_text(p) or "" for p in passages.iterfind("passage")).strip()
+
+        res.add(
+            res.types.MainResult(
+                url=url,
+                title=title,
+                content=content or "",
+            )
+        )
+
+    return res

searx/settings.yml

@@ -41,9 +41,9 @@ search:
   # Filter results. 0: None, 1: Moderate, 2: Strict
   safe_search: 0
   # Existing autocomplete backends: "360search", "baidu", "bing", "brave", "dbpedia", "duckduckgo", "google",
-  # "kagi", "yandex", "privacywall", "mwmbl", "naver", "seznam", "sogou", "startpage", "swisscows", "quark",
+  # "yandex", "privacywall", "mwmbl", "naver", "seznam", "sogou", "startpage", "swisscows", "quark", "qwant",
-  # "qwant", "wikipedia" - leave blank to turn it off by default.
+  # "wikipedia" - set blank to turn it off by default.
-  autocomplete: ""
+  autocomplete: "duckduckgo"
   # minimun characters to type before autocompleter starts
   autocomplete_min: 4
   # backend for the favicon near URL in search results.
@@ -107,11 +107,10 @@ server:
   image_proxy: false
   # 1.0 and 1.1 are supported
   http_protocol_version: "1.0"
-  # POST queries are "more secure!" but are also the source of hard-to-locate
+  # POST keeps search queries out of the browsers history, but causes UX issues.
-  # annoyances, which is why GET may be better for end users and their browsers.
   # see https://github.com/searxng/searxng/pull/3619
   # Is overwritten by ${SEARXNG_METHOD}
-  method: "POST"
+  method: "GET"
   default_http_headers:
     X-Content-Type-Options: nosniff
     X-Download-Options: noopen
@@ -853,28 +852,28 @@ engines:
     shortcut: dog
     dogpile_categ: search
     categories: general
-    disabled: true
+    inactive: true
 
   - name: dogpile images
     engine: dogpile
     shortcut: dogi
     dogpile_categ: images
     categories: images
-    disabled: true
+    inactive: true
 
   - name: dogpile videos
     engine: dogpile
     shortcut: dogv
     dogpile_categ: videos
     categories: videos
-    disabled: true
+    inactive: true
 
   - name: dogpile news
     engine: dogpile
     shortcut: dogn
     dogpile_categ: news
     categories: news
-    disabled: true
+    inactive: true
 
   # duckduckgo uses html.duckduckgo.com,
   # duckduckgo web uses duckduckgo.com
@@ -2585,6 +2584,16 @@ engines:
     shortcut: ydi
     disabled: true
 
+  - name: yandex api
+    engine: yandex_api
+    categories: [general, web]
+    shortcut: yda
+    inactive: true
+    # api_key: ""
+    # folder_id: ""
+    # search_type: SEARCH_TYPE_COM
+    # l10n: LOCALIZATION_EN
+
   - name: yandex music
     engine: yandex_music
     network: yandex

searx/settings_defaults.py

@@ -192,7 +192,7 @@ SCHEMA: dict[str, t.Any] = {
     'brand': SettingsBrand,
     'search': {
         'safe_search': SettingsValue((0, 1, 2), 0),
-        'autocomplete': SettingsValue(str, ''),
+        'autocomplete': SettingsValue(str, 'duckduckgo'),
         'autocomplete_min': SettingsValue(int, 4),
         'favicon_resolver': SettingsValue(str, ''),
         'default_lang': SettingsValue(tuple(SXNG_LOCALE_TAGS + ['']), ''),
@@ -219,7 +219,7 @@ SCHEMA: dict[str, t.Any] = {
         'base_url': SettingsValue((False, str), False, 'SEARXNG_BASE_URL'),
         'image_proxy': SettingsValue(bool, False, 'SEARXNG_IMAGE_PROXY'),
         'http_protocol_version': SettingsValue(('1.0', '1.1'), '1.0'),
-        'method': SettingsValue(('POST', 'GET'), 'POST', 'SEARXNG_METHOD'),
+        'method': SettingsValue(('POST', 'GET'), 'GET', 'SEARXNG_METHOD'),
         'default_http_headers': SettingsValue(dict, {}),
     },
     # redis is deprecated ..

searx/templates/simple/elements/apis.html

@@ -4,7 +4,7 @@
     <div class="wrapper">
       {%- for output_type in search_formats -%}
         <div class="left">
-          <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">
+          <form method="{{ method }}" action="{{ url_for('search') }}">
             <input type="hidden" name="q" value="{{ q|e }}">
             {%- for category in selected_categories -%}
               <input type="hidden" name="category_{{ category }}" value="1">

searx/templates/simple/elements/corrections.html

@@ -2,7 +2,7 @@
       <h4 id="corrections-title">{{ _('Try searching for:') }}</h4>
       {% for correction in corrections %}
       <div class="left">
-	      <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" role="navigation">
+	      <form method="{{ method }}" action="{{ url_for('search') }}" role="navigation">
           {% for category in selected_categories %}
           <input type="hidden" name="category_{{ category }}" value="1">
           {% endfor %}

searx/templates/simple/elements/infobox.html

@@ -31,7 +31,7 @@
         <div>
           <h3><bdi>{{ topic.name }}</bdi></h3>
           {%- for suggestion in topic.suggestions -%}
-            <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">
+            <form method="{{ method }}" action="{{ url_for('search') }}">
               <input type="hidden" name="q" value="{{ suggestion }}">
               <input type="hidden" name="time_range" value="{{ time_range }}">
               <input type="hidden" name="language" value="{{ current_language }}">

searx/templates/simple/elements/suggestions.html

@@ -3,7 +3,7 @@
     <summary class="title" id="suggestions-title">{{ _('Suggestions') }}</summary>
     <ul class="wrapper">
       {%- for suggestion in suggestions -%}
-      <li><form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">
+      <li><form method="{{ method }}" action="{{ url_for('search') }}">
           <input type="hidden" name="q" value="{{ suggestion.url }}">
           {%- for category in selected_categories -%}
             <input type="hidden" name="category_{{ category }}" value="1">

searx/templates/simple/results.html

@@ -75,7 +75,7 @@
     {% if paging %}
     <nav id="pagination" role="navigation">
         {% if pageno > 1 %}
-            <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="previous_page">
+            <form method="{{ method }}" action="{{ url_for('search') }}" class="previous_page">
                 <div class="{% if rtl %}right{% else %}left{% endif %}">
                   <input type="hidden" name="q" value="{{ q|e }}" >
                   {% for category in selected_categories %}
@@ -93,7 +93,7 @@
             </form>
         {% endif %}
         {%- if results | count > 0 -%}
-          <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="next_page">
+          <form method="{{ method }}" action="{{ url_for('search') }}" class="next_page">
               <div class="{% if rtl %}left{% else %}right{% endif %}">
                 <input type="hidden" name="q" value="{{ q|e }}" >
                 {% for category in selected_categories %}
@@ -119,7 +119,7 @@
 
         <div class="numbered_pagination">
         {% for x in range(pstart, pend) %}
-            <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="page_number">
+            <form method="{{ method }}" action="{{ url_for('search') }}" class="page_number">
                 <input type="hidden" name="q" value="{{ q|e }}" >
                 {% for category in selected_categories %}
                 <input type="hidden" name="category_{{ category }}" value="1" >

searx/templates/simple/search.html

@@ -1,4 +1,4 @@
-<form id="search" method="{{ method or 'POST' }}" action="{{ url_for('search') }}" role="search">
+<form id="search" method="{{ method }}" action="{{ url_for('search') }}" role="search">
   <div id="search_header">
     <a id="search_logo" href="{{ url_for('index') }}" tabindex="0" title="{{ _('Display the front page') }}">
       <span hidden>SearXNG</span>

searx/templates/simple/simple_search.html

@@ -1,4 +1,4 @@
-<form id="search" method="{{ method or 'POST' }}" action="{{ url_for('search') }}" role="search">
+<form id="search" method="{{ method }}" action="{{ url_for('search') }}" role="search">
   <div id="search_header">
     <div id="search_view">
       <div class="search_box">

searx/webapp.py

@@ -1195,7 +1195,7 @@ def opensearch():
         method = 'GET'
 
     if method not in ('POST', 'GET'):
-        method = 'POST'
+        method = 'GET'
 
     ret = render('opensearch.xml', opensearch_method=method, autocomplete=autocomplete)
     resp = Response(response=ret, status=200, mimetype="application/opensearchdescription+xml")