[fix] clarify the mess of Engine.setup and Engine.init (#6343)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/enginelib/__init__.py

@@ -361,37 +361,43 @@ class Engine(abc.ABC):  # pylint: disable=too-few-public-methods
          https: socks5://proxy:port
     """
 
-    def setup(self, engine_settings: dict[str, t.Any]) -> bool:  # pylint: disable=unused-argument
+    def setup(self, engine_settings: dict[str, t.Any]) -> bool | None:  # pylint: disable=unused-argument
         """Dynamic setup of the engine settings.
 
         With this method, the engine's setup is carried out.  For example, to
         check or dynamically adapt the values handed over in the parameter
-        ``engine_settings``.  The return value (True/False) indicates whether
-        the setup was successful and the engine can be built or rejected.
+        ``engine_settings``.
 
-        The method is optional and is called synchronously as part of the
+        Whether the initialization was successful can be indicated by the return
+        value ``True`` or even ``False``.
+
+        - If no return value (``None`` ) is given from this method , this is
+          equivalent to ``True``.
+
+        - If an exception is thrown as part of the initialization, this is
+          equivalent to ``False``.
+
+        The method is optional and is called **synchronously** as part of the
         initialization of the service and is therefore only suitable for simple
-        (local) exams/changes at the engine setting.  The :py:obj:`Engine.init`
-        method must be used for longer tasks in which values of a remote must be
-        determined, for example.
+        (local) exams/changes at the engine setting.
+
+        The :py:obj:`Engine.init` method must be used for longer tasks in which
+        values of a remote must be determined, for example.
         """
         return True
 
     def init(self, engine_settings: dict[str, t.Any]) -> bool | None:  # pylint: disable=unused-argument
         """Initialization of the engine.
 
-        The method is optional and asynchronous (in a thread).  It is suitable,
-        for example, for setting up a cache (for the engine) or for querying
-        values (required by the engine) from a remote.
+        The method is optional and called **asynchronous** (in a thread).  The
+        method is comparable to :py:obj:`Engine.setup`, it is suitable, for
+        caching data that first needs to be requested from a remote.
 
-        Whether the initialization was successful can be indicated by the return
-        value ``True`` or even ``False``.
+        The method is optional and runs **asynchronously** (in a thread), it is
+        comparable to :py:obj:`Engine.setup`. For instance, it is suitable for
+        caching data that first needs to be requested from a remote source.
 
-        - If no return value is given from this init method (``None``), this is
-          equivalent to ``True``.
-
-        - If an exception is thrown as part of the initialization, this is
-          equivalent to ``False``.
+        The evaluation of the return value is analogous to :py:obj:`Engine.setup`.
         """
         return True
 

searx/engines/__init__.py

@@ -269,21 +269,27 @@ def is_engine_active(engine: "Engine | types.ModuleType"):
 
 
 def call_engine_setup(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]) -> bool:
-    setup_ok = False
+
+    setup_ok: bool | None = False
     setup_func = getattr(engine, "setup", None)
 
     if setup_func is None:
         setup_ok = True
     elif not callable(setup_func):
-        logger.error("engine's setup method isn't a callable (is of type: %s)", type(setup_func))
+        logger.error(f"engine's setup method isn't a callable (is of type: {type(setup_func)})")
     else:
         try:
             setup_ok = engine.setup(engine_data)
         except Exception as e:  # pylint: disable=broad-except
-            logger.exception('exception : {0}'.format(e))
+            logger.exception(f"(PID {os.getpid()}) {engine.name}: engine SETUP failed, exception: {e}")
+            setup_ok = False
+
+    # The evaluation of the return value is analogous to Engine.init
+    if setup_ok is None:
+        setup_ok = True
 
     if not setup_ok:
-        logger.error("%s: Engine setup was not successful, engine is set to inactive.", engine.name)
+        logger.error(f"(PID {os.getpid()}) {engine.name}: engine setup was not successful")
     return setup_ok
 
 
@@ -311,14 +317,16 @@ def load_engines(engine_list: list[dict[str, t.Any]]):
     for engine_data in engine_list:
         if engine_data.get("inactive") is True:
             continue
+
         engine = load_engine(engine_data)
+
         if engine:
             register_engine(engine)
         else:
             # if an engine can't be loaded (if for example the engine is missing
             # tor or some other requirements) its set to inactive!
             logger.error(
-                f"(PID {os.getpid()}) loading engine %s failed: set engine to inactive!", engine_data.get("name", "???")
+                f"(PID {os.getpid()}) {engine_data.get('name', '???')}: can't register engine (loading engine failed)"
             )
             engine_data["inactive"] = True
     return engines

searx/engines/heexy.py

@@ -13,9 +13,13 @@ It seems to use Bing internally, as the image thumbnails are loaded from Bing.
 from urllib.parse import urlencode
 
 import typing as t
+from lxml import html
 
-from searx.exceptions import SearxEngineAccessDeniedException
+from searx.enginelib import EngineCache
+from searx.network import get
+from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
 from searx.result_types import EngineResults
+from searx.utils import eval_xpath, extract_text, gen_useragent
 
 if t.TYPE_CHECKING:
     from searx.extended_types import SXNG_Response
@@ -38,14 +42,45 @@ heexy_categ = "web"
 """Category to search in. Can be either "web" or "image"."""
 
 
-base_url = "https://seapi.heexy.org"
+base_url = "https://heexy.org"
+api_url = "https://seapi.heexy.org"
 safe_search_map = {0: "off", 1: "on", 2: "on"}
 
+CACHE: EngineCache
+"""Cache for storing the ``X-Data-Cacheft`` token (acts like an API key)."""
+
+
+def setup(engine_settings: dict[str, t.Any]) -> bool:
+    global CACHE  # pylint: disable=global-statement
 
-def init(_):
     if heexy_categ not in ("web", "image"):
         raise ValueError("invalid search category: %s" % heexy_categ)
 
+    CACHE = EngineCache(engine_settings["name"])
+    return True
+
+
+def _get_api_token(query: str) -> str:
+    """The API token is independent of the search query. We just need any query
+    to obtain it initially, and don't hardcode it here to decrease chances of
+    getting blocked. The token must be passed as ``X-Data-Cacheft`` header."""
+
+    cached_token: str = CACHE.get("token")
+    if cached_token:
+        return cached_token
+
+    resp = get(f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent()})
+    if not resp.ok:
+        raise SearxEngineAPIException("failed to obtain request token: invalid response code")
+
+    doc = html.fromstring(resp.text)
+    token = extract_text(eval_xpath(doc, "//html/@data-cacheft"))
+    if not token:
+        raise SearxEngineAPIException("failed to obtain request token: no token found")
+
+    CACHE.set("token", token)
+    return token
+
 
 def request(query: str, params: "OnlineParams") -> None:
     args = {
@@ -56,8 +91,9 @@ def request(query: str, params: "OnlineParams") -> None:
     if params["searxng_locale"] != "all":
         args["lang"] = params["searxng_locale"].split("-")[0]
 
-    params["url"] = f"{base_url}/search/{heexy_categ}?{urlencode(args)}"
-    params["headers"]["Origin"] = base_url
+    params["url"] = f"{api_url}/search/{heexy_categ}?{urlencode(args)}"
+    params["headers"]["X-Data-Cacheft"] = _get_api_token(query)
+    params["headers"]["Origin"] = api_url
 
 
 def response(resp: "SXNG_Response"):

searx/engines/qwant.py

@@ -38,6 +38,7 @@ Implementations
 
 """
 
+import random
 import typing as t
 
 from datetime import (
@@ -89,6 +90,11 @@ qwant_categ: str = None  # pyright: ignore[reportAssignmentType]
 
 safesearch = True
 
+# tgp seems to be short for "test group" - its actual value doesn't matter, as
+# long as it's sent and at the correct position in the query params and doesn't
+# change too frequently
+test_group_value = random.randint(1, 3)
+
 # fmt: off
 qwant_news_locales = [
     "ca_ad", "ca_es", "ca_fr", "co_fr", "de_at", "de_ch", "de_de", "en_au",
@@ -114,17 +120,24 @@ def request(query: str, params: "OnlineParams") -> None:
     results_per_page = 10
     if qwant_categ == "images":
         results_per_page = 50
+
     args = {
         "q": query,
         "count": results_per_page,
         "locale": q_locale,
         "offset": (params["pageno"] - 1) * results_per_page,
+        "tgp": test_group_value,
         "device": "desktop",
         "safesearch": params["safesearch"],
-        "tgp": 1,
         "display": True,
         "llm": True,
     }
+
+    # shuffle query parameters to be harder to fingerprint
+    args = list(args.items())
+    random.shuffle(args)
+    args = dict(args)
+
     params["raise_for_httperror"] = False
 
     params["url"] = f"{api_url}{qwant_categ}?{urlencode(args)}"
@@ -190,7 +203,6 @@ def response(resp: "SXNG_Response") -> EngineResults:
 
         mainline_items: list[dict[str, t.Any]] = row.get("items", [])
         for item in mainline_items:
-
             title: str = item.get("title", "")
             res_url: str = item.get("url", "")
             pub_date: datetime | None = None

searx/search/processors/__init__.py

@@ -16,6 +16,7 @@ __all__ = [
 
 import typing as t
 
+import os
 from searx import logger
 from searx import engines
 
@@ -92,7 +93,9 @@ class ProcessorMap(dict[str, EngineProcessor]):
             self[eng_proc.engine.name] = eng_proc
             # logger.debug("registered engine processor: %s", eng_proc.engine.name)
         else:
-            logger.error("can't register engine processor: %s (init failed)", eng_proc.engine.name)
+            logger.error(
+                f"(PID {os.getpid()}) {eng_proc.engine.name}: can't register engines processor (init engine failed)"
+            )
 
         return eng_proc_ok
 

searx/search/processors/abstract.py

@@ -150,19 +150,26 @@ class EngineProcessor(ABC):
         threading.Thread(target=__init_processor_thread, daemon=True).start()
 
     def init_engine(self) -> bool:
+
         eng_setting = get_engine_from_settings(self.engine.name)
         init_ok: bool | None = False
+
         try:
             init_ok = self.engine.init(eng_setting)
-        except Exception:  # pylint: disable=broad-except
-            logger.exception(
-                f"(PID {os.getpid()}) Init method of engine %s failed due to an exception.", self.engine.name
-            )
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception(f"(PID {os.getpid()}) {self.engine.name}: engine INIT failed, exception: {e}")
             init_ok = False
+
         # In older engines, None is returned from the init method, which is
-        # equivalent to indicating that the initialization was successful.
+        # equivalent to indicating that the initialization was successful
+        # (compare: Engine.setup).
+
         if init_ok is None:
             init_ok = True
+
+        if not init_ok:
+            logger.error(f"(PID {os.getpid()}) {self.engine.name}: engine init was not successful")
+
         return init_ok
 
     def handle_exception(

searx/settings.yml

@@ -3311,6 +3311,36 @@ engines:
     base_url: https://info.searchtoday.site
     disabled: true
 
+  - name: sina
+    engine: json_engine
+    shortcut: sina
+    categories: news
+    paging: true
+    first_page_num: 1
+    search_url: https://search.sina.com.cn/api/search?q={query}&tp=mix&sort=0&page={pageno}&size=10&from=search_result
+    results_query: data/list
+    url_query: url
+    title_query: title
+    content_query: intro
+    thumbnail_query: thumb
+    title_html_to_text: true
+    time_range_map:
+      day: d
+      week: w
+      month: m
+      year: y
+    time_range_support: true
+    time_range_url: "&from=advanced_search&time={time_range_val}"
+    disabled: true
+    inactive: true
+    language: zh
+    about:
+      website: https://search.sina.com.cn
+      wikidata_id: Q938668
+      use_official_api: false
+      require_api_key: false
+      results: JSON
+
   # - name: webcrawler
   #   engine: s1search
   #   shortcut: wc