[mod] addition of various type hints / tbc

- pyright configuration [1]_ - stub files: types-lxml [2]_ - addition of various type hints - enable use of new type system features on older Python versions [3]_ - ``.tool-versions`` - set python to lowest version we support (3.10.18) [4]_: Older versions typically lack some typing features found in newer Python versions. Therefore, for local type checking (before commit), it is necessary to use the older Python interpreter. .. [1] https://docs.basedpyright.com/v1.20.0/configuration/config-files/ .. [2] https://pypi.org/project/types-lxml/ .. [3] https://typing-extensions.readthedocs.io/en/latest/# .. [4] https://mise.jdx.dev/configuration.html#tool-versions Signed-off-by: Markus Heiser <markus.heiser@darmarit.de> Format: reST
2025-12-22 19:50:00 +00:00 · 2025-08-22 17:17:51 +02:00
parent 09500459fe
commit 57b9673efb
107 changed files with 1205 additions and 1251 deletions
--- a/searx/enginelib/init.py
+++ b/searx/enginelib/init.py
@@ -22,21 +22,25 @@ an example in which the command line is called in the development environment::
 -----

 """
-from __future__ import annotations

 __all__ = ["EngineCache", "Engine", "ENGINES_CACHE"]

-from typing import List, Callable, TYPE_CHECKING, Any
+import typing as t
+import abc
+from collections.abc import Callable
+import logging
 import string
 import typer

-from ..cache import ExpireCache, ExpireCacheCfg
+from ..cache import ExpireCacheSQLite, ExpireCacheCfg

-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
    from searx.enginelib import traits
+    from searx.enginelib.traits import EngineTraits
+    from searx.extended_types import SXNG_Response
+    from searx.result_types import EngineResults

-
-ENGINES_CACHE = ExpireCache.build_cache(
+ENGINES_CACHE: ExpireCacheSQLite = ExpireCacheSQLite.build_cache(
    ExpireCacheCfg(
        name="ENGINES_CACHE",
        MAXHOLD_TIME=60 * 60 * 24 * 7,  # 7 days
@@ -62,7 +66,7 @@ def state():
    title = f"properties of {ENGINES_CACHE.cfg.name}"
    print(title)
    print("=" * len(title))
-    print(str(ENGINES_CACHE.properties))  # type: ignore
+    print(str(ENGINES_CACHE.properties))


@app.command()
@@ -152,11 +156,11 @@ class EngineCache:
    """

    def __init__(self, engine_name: str, expire: int | None = None):
-        self.expire = expire or ENGINES_CACHE.cfg.MAXHOLD_TIME
+        self.expire: int = expire or ENGINES_CACHE.cfg.MAXHOLD_TIME
        _valid = "-_." + string.ascii_letters + string.digits
-        self.table_name = "".join([c if c in _valid else "_" for c in engine_name])
+        self.table_name: str = "".join([c if c in _valid else "_" for c in engine_name])

-    def set(self, key: str, value: Any, expire: int | None = None) -> bool:
+    def set(self, key: str, value: t.Any, expire: int | None = None) -> bool:
        return ENGINES_CACHE.set(
            key=key,
            value=value,
@@ -164,14 +168,14 @@ class EngineCache:
            ctx=self.table_name,
        )

-    def get(self, key: str, default=None) -> Any:
+    def get(self, key: str, default: t.Any = None) -> t.Any:
        return ENGINES_CACHE.get(key, default=default, ctx=self.table_name)

    def secret_hash(self, name: str | bytes) -> str:
        return ENGINES_CACHE.secret_hash(name=name)


-class Engine:  # pylint: disable=too-few-public-methods
+class Engine(abc.ABC):  # pylint: disable=too-few-public-methods
    """Class of engine instances build from YAML settings.

    Further documentation see :ref:`general engine configuration`.
@@ -181,6 +185,8 @@ class Engine:  # pylint: disable=too-few-public-methods
       This class is currently never initialized and only used for type hinting.
    """

+    logger: logging.Logger
+
    # Common options in the engine module

    engine_type: str
@@ -220,15 +226,15 @@ class Engine:  # pylint: disable=too-few-public-methods
        region: fr-BE
    """

-    fetch_traits: Callable
+    fetch_traits: "Callable[[EngineTraits, bool], None]"
    """Function to to fetch engine's traits from origin."""

-    traits: traits.EngineTraits
+    traits: "traits.EngineTraits"
    """Traits of the engine."""

    # settings.yml

-    categories: List[str]
+    categories: list[str]
    """Specifies to which :ref:`engine categories` the engine should be added."""

    name: str
@@ -269,7 +275,7 @@ class Engine:  # pylint: disable=too-few-public-methods
    inactive: bool
    """Remove the engine from the settings (*disabled & removed*)."""

-    about: dict
+    about: dict[str, dict[str, str]]
    """Additional fields describing the engine.

    .. code:: yaml
@@ -291,9 +297,21 @@ class Engine:  # pylint: disable=too-few-public-methods
    the user is used to build and send a ``Accept-Language`` header in the
    request to the origin search engine."""

-    tokens: List[str]
+    tokens: list[str]
    """A list of secret tokens to make this engine *private*, more details see
    :ref:`private engines`."""

    weight: int
    """Weighting of the results of this engine (:ref:`weight <settings engines>`)."""
+
+    def init(self, engine_settings: dict[str, t.Any]) -> None:  # pyright: ignore[reportUnusedParameter]
+        """Initialization of the engine. If no initialization is needed, drop
+        this init function."""
+
+    @abc.abstractmethod
+    def request(self, query: str, params: dict[str, t.Any]) -> None:
+        """Build up the params for the online request."""
+
+    @abc.abstractmethod
+    def response(self, resp: "SXNG_Response") -> "EngineResults":
+        """Parse out the result items from the response."""
--- a/searx/enginelib/traits.py
+++ b/searx/enginelib/traits.py
@@ -15,12 +15,12 @@ import os
 import json
 import dataclasses
 import types
-from typing import Dict, Literal, Iterable, Union, Callable, Optional, TYPE_CHECKING
-
+import typing as t
+import pathlib
 from searx import locales
 from searx.data import data_dir, ENGINE_TRAITS

-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
    from . import Engine


@@ -28,7 +28,7 @@ class EngineTraitsEncoder(json.JSONEncoder):
    """Encodes :class:`EngineTraits` to a serializable object, see
    :class:`json.JSONEncoder`."""

-    def default(self, o):
+    def default(self, o: t.Any) -> t.Any:
        """Return dictionary of a :class:`EngineTraits` object."""
        if isinstance(o, EngineTraits):
            return o.__dict__
@@ -39,7 +39,7 @@ class EngineTraitsEncoder(json.JSONEncoder):
 class EngineTraits:
    """The class is intended to be instantiated for each engine."""

-    regions: Dict[str, str] = dataclasses.field(default_factory=dict)
+    regions: dict[str, str] = dataclasses.field(default_factory=dict)
    """Maps SearXNG's internal representation of a region to the one of the engine.

    SearXNG's internal representation can be parsed by babel and the value is
@@ -56,7 +56,7 @@ class EngineTraits:
          ...
    """

-    languages: Dict[str, str] = dataclasses.field(default_factory=dict)
+    languages: dict[str, str] = dataclasses.field(default_factory=dict)
    """Maps SearXNG's internal representation of a language to the one of the engine.

    SearXNG's internal representation can be parsed by babel and the value is
@@ -73,20 +73,20 @@ class EngineTraits:
          ...
    """

-    all_locale: Optional[str] = None
+    all_locale: str | None = None
    """To which locale value SearXNG's ``all`` language is mapped (shown a "Default
    language").
    """

-    data_type: Literal['traits_v1'] = 'traits_v1'
+    data_type: t.Literal['traits_v1'] = 'traits_v1'
    """Data type, default is 'traits_v1'.
    """

-    custom: Dict[str, Union[Dict[str, Dict], Iterable[str]]] = dataclasses.field(default_factory=dict)
+    custom: dict[str, t.Any] = dataclasses.field(default_factory=dict)
    """A place to store engine's custom traits, not related to the SearXNG core.
    """

-    def get_language(self, searxng_locale: str, default=None):
+    def get_language(self, searxng_locale: str, default: t.Any = None):
        """Return engine's language string that *best fits* to SearXNG's locale.

        :param searxng_locale: SearXNG's internal representation of locale
@@ -102,7 +102,7 @@ class EngineTraits:
            return self.all_locale
        return locales.get_engine_locale(searxng_locale, self.languages, default=default)

-    def get_region(self, searxng_locale: str, default=None):
+    def get_region(self, searxng_locale: str, default: t.Any = None) -> t.Any:
        """Return engine's region string that best fits to SearXNG's locale.

        :param searxng_locale: SearXNG's internal representation of locale
@@ -133,10 +133,10 @@ class EngineTraits:

    def copy(self):
        """Create a copy of the dataclass object."""
-        return EngineTraits(**dataclasses.asdict(self))
+        return EngineTraits(**dataclasses.asdict(self))  # type: ignore

    @classmethod
-    def fetch_traits(cls, engine: Engine) -> Union['EngineTraits', None]:
+    def fetch_traits(cls, engine: "Engine | types.ModuleType") -> "EngineTraits | None":
        """Call a function ``fetch_traits(engine_traits)`` from engines namespace to fetch
        and set properties from the origin engine in the object ``engine_traits``.  If
        function does not exists, ``None`` is returned.
@@ -150,7 +150,7 @@ class EngineTraits:
            fetch_traits(engine_traits)
        return engine_traits

-    def set_traits(self, engine: Engine):
+    def set_traits(self, engine: "Engine | types.ModuleType"):
        """Set traits from self object in a :py:obj:`.Engine` namespace.

        :param engine: engine instance build by :py:func:`searx.engines.load_engine`
@@ -161,14 +161,14 @@ class EngineTraits:
        else:
            raise TypeError('engine traits of type %s is unknown' % self.data_type)

-    def _set_traits_v1(self, engine: Engine):
+    def _set_traits_v1(self, engine: "Engine | types.ModuleType"):
        # For an engine, when there is `language: ...` in the YAML settings the engine
        # does support only this one language (region)::
        #
        #   - name: google italian
        #     engine: google
        #     language: it
-        #     region: it-IT                                      # type: ignore
+        #     region: it-IT

        traits = self.copy()

@@ -186,16 +186,16 @@ class EngineTraits:
                raise ValueError(_msg % (engine.name, 'region', engine.region))
            traits.regions = {engine.region: regions[engine.region]}

-        engine.language_support = bool(traits.languages or traits.regions)
+        engine.language_support = bool(traits.languages or traits.regions)  # type: ignore

        # set the copied & modified traits in engine's namespace
-        engine.traits = traits
+        engine.traits = traits  # pyright: ignore[reportAttributeAccessIssue]


-class EngineTraitsMap(Dict[str, EngineTraits]):
+class EngineTraitsMap(dict[str, EngineTraits]):
    """A python dictionary to map :class:`EngineTraits` by engine name."""

-    ENGINE_TRAITS_FILE = (data_dir / 'engine_traits.json').resolve()
+    ENGINE_TRAITS_FILE: pathlib.Path = (data_dir / 'engine_traits.json').resolve()
    """File with persistence of the :py:obj:`EngineTraitsMap`."""

    def save_data(self):
@@ -212,7 +212,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):
        return obj

    @classmethod
-    def fetch_traits(cls, log: Callable) -> 'EngineTraitsMap':
+    def fetch_traits(cls, log: t.Callable[[str], None]) -> 'EngineTraitsMap':
        from searx import engines  # pylint: disable=cyclic-import, import-outside-toplevel

        names = list(engines.engines)
@@ -220,7 +220,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):
        obj = cls()

        for engine_name in names:
-            engine = engines.engines[engine_name]
+            engine: Engine | types.ModuleType = engines.engines[engine_name]
            traits = None

            # pylint: disable=broad-exception-caught
@@ -242,7 +242,7 @@ class EngineTraitsMap(Dict[str, EngineTraits]):

        return obj

-    def set_traits(self, engine: Engine | types.ModuleType):
+    def set_traits(self, engine: "Engine | types.ModuleType"):
        """Set traits in a :py:obj:`Engine` namespace.

        :param engine: engine instance build by :py:func:`searx.engines.load_engine`