jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2025-12-22 19:50:00 +00:00
Code Issues Projects Releases Wiki Activity

[mod] addition of various type hints / tbc

Browse Source 
- pyright configuration [1]_
- stub files: types-lxml [2]_
- addition of various type hints
- enable use of new type system features on older Python versions [3]_
- ``.tool-versions`` - set python to lowest version we support (3.10.18) [4]_:
  Older versions typically lack some typing features found in newer Python
  versions.  Therefore, for local type checking (before commit), it is necessary
  to use the older Python interpreter.

.. [1] https://docs.basedpyright.com/v1.20.0/configuration/config-files/
.. [2] https://pypi.org/project/types-lxml/
.. [3] https://typing-extensions.readthedocs.io/en/latest/#
.. [4] https://mise.jdx.dev/configuration.html#tool-versions

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
Format: reST
This commit is contained in:
Markus Heiser
2025-08-22 17:17:51 +02:00
committed by Markus Heiser
parent 09500459fe
commit 57b9673efb
107 changed files with 1205 additions and 1251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
searx/engines/__builtins__.pyi
View File

@@ -13,10 +13,13 @@ intended monkey patching of the engine modules.
from __future__ import annotations
import logging
from searx.enginelib import traits as _traits
logger: logging.Logger
supported_languages: str
language_aliases: str
language_support: bool
traits: _traits.EngineTraits
# from searx.engines.ENGINE_DEFAULT_ARGS
about: dict[str, dict[str, str | None | bool]]

22
searx/engines/__init__.py
View File

@@ -51,8 +51,8 @@ ENGINE_DEFAULT_ARGS: dict[str, int | str | list[t.Any] | dict[str, t.Any] | bool
# set automatically when an engine does not have any tab category
DEFAULT_CATEGORY = 'other'
categories: dict[str, list[str]] = {'general': []}
engines: dict[str, Engine | types.ModuleType] = {}
categories: "dict[str, list[Engine|types.ModuleType]]" = {'general': []}
engines: "dict[str, Engine | types.ModuleType]" = {}
engine_shortcuts = {}
"""Simple map of registered *shortcuts* to name of the engine (or ``None``).
@@ -76,7 +76,7 @@ def check_engine_module(module: types.ModuleType):
raise TypeError(msg)
def load_engine(engine_data: dict[str, t.Any]) -> Engine | types.ModuleType | None:
def load_engine(engine_data: dict[str, t.Any]) -> "Engine | types.ModuleType | None":
"""Load engine from ``engine_data``.
:param dict engine_data: Attributes from YAML ``settings:engines/<engine>``
@@ -151,7 +151,7 @@ def load_engine(engine_data: dict[str, t.Any]) -> Engine | types.ModuleType | No
return engine
def set_loggers(engine, engine_name):
def set_loggers(engine: "Engine|types.ModuleType", engine_name: str):
# set the logger for engine
engine.logger = logger.getChild(engine_name)
# the engine may have load some other engines
@@ -170,7 +170,7 @@ def set_loggers(engine, engine_name):
module.logger = logger.getChild(module_engine_name) # type: ignore
def update_engine_attributes(engine: Engine | types.ModuleType, engine_data):
def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]):
# set engine attributes from engine_data
for param_name, param_value in engine_data.items():
if param_name == 'categories':
@@ -188,13 +188,13 @@ def update_engine_attributes(engine: Engine | types.ModuleType, engine_data):
setattr(engine, arg_name, copy.deepcopy(arg_value))
def update_attributes_for_tor(engine: Engine | types.ModuleType):
def update_attributes_for_tor(engine: "Engine | types.ModuleType"):
if using_tor_proxy(engine) and hasattr(engine, 'onion_url'):
engine.search_url = engine.onion_url + getattr(engine, 'search_path', '') # type: ignore
engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0) # type: ignore
def is_missing_required_attributes(engine):
def is_missing_required_attributes(engine: "Engine | types.ModuleType"):
"""An attribute is required when its name doesn't start with ``_`` (underline).
Required attributes must not be ``None``.
@@ -207,12 +207,12 @@ def is_missing_required_attributes(engine):
return missing
def using_tor_proxy(engine: Engine | types.ModuleType):
def using_tor_proxy(engine: "Engine | types.ModuleType"):
"""Return True if the engine configuration declares to use Tor."""
return settings['outgoing'].get('using_tor_proxy') or getattr(engine, 'using_tor_proxy', False)
def is_engine_active(engine: Engine | types.ModuleType):
def is_engine_active(engine: "Engine | types.ModuleType"):
# check if engine is inactive
if engine.inactive is True:
return False
@@ -224,7 +224,7 @@ def is_engine_active(engine: Engine | types.ModuleType):
return True
def register_engine(engine: Engine | types.ModuleType):
def register_engine(engine: "Engine | types.ModuleType"):
if engine.name in engines:
logger.error('Engine config error: ambiguous name: {0}'.format(engine.name))
sys.exit(1)
@@ -239,7 +239,7 @@ def register_engine(engine: Engine | types.ModuleType):
categories.setdefault(category_name, []).append(engine)
def load_engines(engine_list):
def load_engines(engine_list: list[dict[str, t.Any]]):
"""usage: ``engine_list = settings['engines']``"""
engines.clear()
engine_shortcuts.clear()

6
searx/engines/adobe_stock.py
View File

@@ -37,17 +37,11 @@ Implementation
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from datetime import datetime, timedelta
from urllib.parse import urlencode
import isodate
if TYPE_CHECKING:
import logging
logger: logging.Logger
about = {
"website": "https://stock.adobe.com/",
"wikidata_id": "Q5977430",

38
searx/engines/annas_archive.py
View File

@@ -32,18 +32,24 @@ Implementations
===============
"""
import typing as t
from typing import List, Dict, Any, Optional
from urllib.parse import urlencode
from lxml import html
from lxml.etree import ElementBase
from searx.utils import extract_text, eval_xpath, eval_xpath_getindex, eval_xpath_list
from searx.enginelib.traits import EngineTraits
from searx.data import ENGINE_TRAITS
from searx.exceptions import SearxEngineXPathException
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
# about
about: Dict[str, Any] = {
about: dict[str, t.Any] = {
"website": "https://annas-archive.org/",
"wikidata_id": "Q115288326",
"official_api_documentation": None,
@@ -53,7 +59,7 @@ about: Dict[str, Any] = {
}
# engine dependent config
categories: List[str] = ["files"]
categories: list[str] = ["files"]
paging: bool = True
# search-url
@@ -85,7 +91,7 @@ aa_ext: str = ''
"""
def init(engine_settings=None): # pylint: disable=unused-argument
def init(engine_settings: dict[str, t.Any]) -> None: # pylint: disable=unused-argument
"""Check of engine's settings."""
traits = EngineTraits(**ENGINE_TRAITS['annas archive'])
@@ -99,8 +105,8 @@ def init(engine_settings=None): # pylint: disable=unused-argument
raise ValueError(f'invalid setting ext: {aa_ext}')
def request(query, params: Dict[str, Any]) -> Dict[str, Any]:
lang = traits.get_language(params["language"], traits.all_locale) # type: ignore
def request(query: str, params: dict[str, t.Any]) -> None:
lang = traits.get_language(params["language"], traits.all_locale)
args = {
'lang': lang,
'content': aa_content,
@@ -112,11 +118,10 @@ def request(query, params: Dict[str, Any]) -> Dict[str, Any]:
# filter out None and empty values
filtered_args = dict((k, v) for k, v in args.items() if v)
params["url"] = f"{base_url}/search?{urlencode(filtered_args)}"
return params
def response(resp) -> List[Dict[str, Optional[str]]]:
results: List[Dict[str, Optional[str]]] = []
def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
dom = html.fromstring(resp.text)
# The rendering of the WEB page is strange; positions of Anna's result page
@@ -126,16 +131,17 @@ def response(resp) -> List[Dict[str, Optional[str]]]:
for item in eval_xpath_list(dom, '//main//div[contains(@class, "js-aarecord-list-outer")]/div'):
try:
results.append(_get_result(item))
kwargs: dict[str, t.Any] = _get_result(item)
except SearxEngineXPathException:
pass
return results
continue
res.add(res.types.LegacyResult(**kwargs))
return res
def _get_result(item):
def _get_result(item: ElementBase) -> dict[str, t.Any]:
return {
'template': 'paper.html',
'url': base_url + extract_text(eval_xpath_getindex(item, './a/@href', 0)),
'url': base_url + eval_xpath_getindex(item, './a/@href', 0),
'title': extract_text(eval_xpath(item, './div//a[starts-with(@href, "/md5")]')),
'authors': [extract_text(eval_xpath_getindex(item, './/a[starts-with(@href, "/search")]', 0))],
'publisher': extract_text(
@@ -160,9 +166,9 @@ def fetch_traits(engine_traits: EngineTraits):
engine_traits.custom['sort'] = []
resp = get(base_url + '/search')
if not resp.ok: # type: ignore
if not resp.ok:
raise RuntimeError("Response from Anna's search page is not OK.")
dom = html.fromstring(resp.text) # type: ignore
dom = html.fromstring(resp.text)
# supported language codes

8
searx/engines/archlinux.py
View File

@@ -8,7 +8,6 @@ Arch Wiki blocks access to it.
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode, urljoin, urlparse
import lxml
import babel
@@ -17,13 +16,6 @@ from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
from searx.enginelib.traits import EngineTraits
from searx.locales import language_tag
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://wiki.archlinux.org/',

8
searx/engines/bing.py
View File

@@ -26,7 +26,6 @@ category for the Chinese market.
"""
# pylint: disable=too-many-branches, invalid-name
from typing import TYPE_CHECKING
import base64
import re
import time
@@ -40,13 +39,6 @@ from searx.locales import language_tag, region_tag
from searx.enginelib.traits import EngineTraits
from searx.exceptions import SearxEngineAPIException
if TYPE_CHECKING:
import logging
logger = logging.getLogger()
traits: EngineTraits
about = {
"website": 'https://www.bing.com',
"wikidata_id": 'Q182496',

12
searx/engines/bing_images.py
View File

@@ -2,26 +2,14 @@
"""Bing-Images: description see :py:obj:`searx.engines.bing`.
"""
# pylint: disable=invalid-name
from typing import TYPE_CHECKING
import json
from urllib.parse import urlencode
from lxml import html
from searx.enginelib.traits import EngineTraits
from searx.engines.bing import set_bing_cookies
from searx.engines.bing import fetch_traits # pylint: disable=unused-import
if TYPE_CHECKING:
import logging
logger = logging.getLogger()
traits: EngineTraits
# about
about = {
"website": 'https://www.bing.com/images',

9
searx/engines/bing_news.py
View File

@@ -9,7 +9,6 @@
# pylint: disable=invalid-name
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from lxml import html
@@ -18,14 +17,6 @@ from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_ge
from searx.enginelib.traits import EngineTraits
from searx.engines.bing import set_bing_cookies
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://www.bing.com/news',

9
searx/engines/bing_videos.py
View File

@@ -3,24 +3,15 @@
"""Bing-Videos: description see :py:obj:`searx.engines.bing`.
"""
from typing import TYPE_CHECKING
import json
from urllib.parse import urlencode
from lxml import html
from searx.enginelib.traits import EngineTraits
from searx.engines.bing import set_bing_cookies
from searx.engines.bing import fetch_traits # pylint: disable=unused-import
from searx.engines.bing_images import time_map
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://www.bing.com/videos',

26
searx/engines/brave.py
View File

@@ -117,7 +117,7 @@ Implementations
"""
from typing import Any, TYPE_CHECKING
import typing as t
from urllib.parse import (
urlencode,
@@ -139,13 +139,7 @@ from searx.utils import (
)
from searx.enginelib.traits import EngineTraits
from searx.result_types import EngineResults
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
from searx.extended_types import SXNG_Response
about = {
"website": 'https://search.brave.com/',
@@ -158,17 +152,19 @@ about = {
base_url = "https://search.brave.com/"
categories = []
brave_category = 'search'
Goggles = Any
brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = 'search'
"""Brave supports common web-search, videos, images, news, and goggles search.
- ``search``: Common WEB search
- ``videos``: search for videos
- ``images``: search for images
- ``news``: search for news
- ``goggles``: Common WEB search with custom rules
- ``goggles``: Common WEB search with custom rules, requires a :py:obj:`Goggles` URL.
"""
Goggles: str = ""
"""This should be a URL ending in ``.goggle``"""
brave_spellcheck = False
"""Brave supports some kind of spell checking. When activated, Brave tries to
fix typos, e.g. it searches for ``food`` when the user queries for ``fooh``. In
@@ -192,7 +188,7 @@ time_range_support = False
"""Brave only supports time-range in :py:obj:`brave_category` ``search`` (UI
category All) and in the goggles category."""
time_range_map = {
time_range_map: dict[str, str] = {
'day': 'pd',
'week': 'pw',
'month': 'pm',
@@ -200,12 +196,12 @@ time_range_map = {
}
def request(query, params):
def request(query: str, params: dict[str, t.Any]) -> None:
# Don't accept br encoding / see https://github.com/searxng/searxng/pull/1787
params['headers']['Accept-Encoding'] = 'gzip, deflate'
args = {
args: dict[str, t.Any] = {
'q': query,
'source': 'web',
}
@@ -254,7 +250,7 @@ def _extract_published_date(published_date_raw):
return None
def response(resp) -> EngineResults:
def response(resp: SXNG_Response) -> EngineResults:
if brave_category in ('search', 'goggles'):
return _parse_search(resp)

8
searx/engines/chinaso.py
View File

@@ -54,8 +54,8 @@ Implementations
"""
import typing as t
import base64
import typing
import secrets
from urllib.parse import urlencode
@@ -78,7 +78,7 @@ time_range_support = True
results_per_page = 10
categories = []
ChinasoCategoryType = typing.Literal['news', 'videos', 'images']
ChinasoCategoryType = t.Literal['news', 'videos', 'images']
"""ChinaSo supports news, videos, images search.
- ``news``: search for news
@@ -91,7 +91,7 @@ In the category ``news`` you can additionally filter by option
chinaso_category = 'news'
"""Configure ChinaSo category (:py:obj:`ChinasoCategoryType`)."""
ChinasoNewsSourceType = typing.Literal['CENTRAL', 'LOCAL', 'BUSINESS', 'EPAPER', 'all']
ChinasoNewsSourceType = t.Literal['CENTRAL', 'LOCAL', 'BUSINESS', 'EPAPER', 'all']
"""Filtering ChinaSo-News results by source:
- ``CENTRAL``: central publication
@@ -111,7 +111,7 @@ base_url = "https://www.chinaso.com"
def init(_):
if chinaso_category not in ('news', 'videos', 'images'):
raise ValueError(f"Unsupported category: {chinaso_category}")
if chinaso_category == 'news' and chinaso_news_source not in typing.get_args(ChinasoNewsSourceType):
if chinaso_category == 'news' and chinaso_news_source not in t.get_args(ChinasoNewsSourceType):
raise ValueError(f"Unsupported news source: {chinaso_news_source}")

9
searx/engines/dailymotion.py
View File

@@ -10,8 +10,6 @@ Dailymotion (Videos)
"""
from typing import TYPE_CHECKING
from datetime import datetime, timedelta
from urllib.parse import urlencode
import time
@@ -23,13 +21,6 @@ from searx.exceptions import SearxEngineAPIException
from searx.locales import region_tag, language_tag
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://www.dailymotion.com',

19
searx/engines/demo_offline.py
View File

@@ -12,13 +12,14 @@ close to the implementation, its just a simple example. To get in use of this
"""
import typing as t
import json
from searx.result_types import EngineResults
from searx.enginelib import EngineCache
engine_type = 'offline'
categories = ['general']
engine_type = "offline"
categories = ["general"]
disabled = True
timeout = 2.0
@@ -38,13 +39,13 @@ CACHE: EngineCache
seconds."""
def init(engine_settings):
def init(engine_settings: dict[str, t.Any]) -> None:
"""Initialization of the (offline) engine. The origin of this demo engine is a
simple json string which is loaded in this example while the engine is
initialized."""
global _my_offline_engine, CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"]) # type:ignore
CACHE = EngineCache(engine_settings["name"])
_my_offline_engine = (
'[ {"value": "%s"}'
@@ -55,20 +56,22 @@ def init(engine_settings):
)
def search(query, request_params) -> EngineResults:
def search(query: str, params: dict[str, t.Any]) -> EngineResults:
"""Query (offline) engine and return results. Assemble the list of results
from your local engine. In this demo engine we ignore the 'query' term,
usual you would pass the 'query' term to your local engine to filter out the
results.
"""
res = EngineResults()
count = CACHE.get("count", 0)
for row in json.loads(_my_offline_engine):
count: int = CACHE.get("count", 0)
data_rows: list[dict[str, str]] = json.loads(_my_offline_engine)
for row in data_rows:
count += 1
kvmap = {
'query': query,
'language': request_params['searxng_locale'],
'language': params['searxng_locale'],
'value': row.get("value"),
}
res.add(

71
searx/engines/demo_online.py
View File

@@ -15,29 +15,35 @@ list in ``settings.yml``:
"""
import typing as t
from json import loads
from urllib.parse import urlencode
from searx.result_types import EngineResults
engine_type = 'online'
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
engine_type = "online"
send_accept_language_header = True
categories = ['general']
categories = ["general"]
disabled = True
timeout = 2.0
categories = ['images']
categories = ["images"]
paging = True
page_size = 20
search_api = 'https://api.artic.edu/api/v1/artworks/search?'
image_api = 'https://www.artic.edu/iiif/2/'
search_api = "https://api.artic.edu/api/v1/artworks/search?"
image_api = "https://www.artic.edu/iiif/2/"
about = {
"website": 'https://www.artic.edu',
"wikidata_id": 'Q239303',
"official_api_documentation": 'http://api.artic.edu/docs/',
"website": "https://www.artic.edu",
"wikidata_id": "Q239303",
"official_api_documentation": "http://api.artic.edu/docs/",
"use_official_api": True,
"require_api_key": False,
"results": 'JSON',
"results": "JSON",
}
@@ -45,33 +51,30 @@ about = {
_my_online_engine = None
def init(engine_settings):
def init(engine_settings: dict[str, t.Any]) -> None:
"""Initialization of the (online) engine. If no initialization is needed, drop
this init function.
"""
this init function."""
global _my_online_engine # pylint: disable=global-statement
_my_online_engine = engine_settings.get('name')
_my_online_engine = engine_settings.get("name")
def request(query, params):
def request(query: str, params: dict[str, t.Any]) -> None:
"""Build up the ``params`` for the online request. In this example we build a
URL to fetch images from `artic.edu <https://artic.edu>`__
"""
args = urlencode(
{
'q': query,
'page': params['pageno'],
'fields': 'id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles',
'limit': page_size,
"q": query,
"page": params["pageno"],
"fields": "id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles",
"limit": page_size,
}
)
params['url'] = search_api + args
return params
params["url"] = search_api + args
def response(resp) -> EngineResults:
def response(resp: "SXNG_Response") -> EngineResults:
"""Parse out the result items from the response. In this example we parse the
response from `api.artic.edu <https://artic.edu>`__ and filter out all
images.
@@ -87,20 +90,20 @@ def response(resp) -> EngineResults:
)
)
for result in json_data['data']:
for result in json_data["data"]:
if not result['image_id']:
if not result["image_id"]:
continue
res.append(
{
'url': 'https://artic.edu/artworks/%(id)s' % result,
'title': result['title'] + " (%(date_display)s) // %(artist_display)s" % result,
'content': "%(medium_display)s // %(dimensions)s" % result,
'author': ', '.join(result['artist_titles']),
'img_src': image_api + '/%(image_id)s/full/843,/0/default.jpg' % result,
'template': 'images.html',
}
)
kwargs: dict[str, t.Any] = {
"url": "https://artic.edu/artworks/%(id)s" % result,
"title": result["title"] + " (%(date_display)s) // %(artist_display)s" % result,
"content": "%(medium_display)s // %(dimensions)s" % result,
"author": ", ".join(result["artist_titles"]),
"img_src": image_api + "/%(image_id)s/full/843,/0/default.jpg" % result,
"template": "images.html",
}
res.add(res.types.LegacyResult(**kwargs))
return res

10
searx/engines/duckduckgo.py
View File

@@ -4,11 +4,8 @@ DuckDuckGo WEB
~~~~~~~~~~~~~~
"""
from __future__ import annotations
import json
import re
import typing
from urllib.parse import quote_plus
@@ -31,13 +28,6 @@ from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineCaptchaException
from searx.result_types import EngineResults
if typing.TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://lite.duckduckgo.com/lite/',
"wikidata_id": 'Q12805',

7
searx/engines/duckduckgo_definitions.py
View File

@@ -13,8 +13,6 @@ most of the features are based on English terms.
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode, urlparse, urljoin
from lxml import html
@@ -23,11 +21,6 @@ from searx.utils import extract_text, html_to_text, get_string_replaces_function
from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
from searx.result_types import EngineResults
if TYPE_CHECKING:
import logging
logger: logging.Logger
# about
about = {
"website": 'https://duckduckgo.com/',

11
searx/engines/duckduckgo_extra.py
View File

@@ -4,23 +4,12 @@ DuckDuckGo Extra (images, videos, news)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""
from __future__ import annotations
from datetime import datetime
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from searx.utils import get_embeded_stream_url, html_to_text
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
from searx.engines.duckduckgo import get_ddg_lang, get_vqd
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {

9
searx/engines/duckduckgo_weather.py
View File

@@ -3,7 +3,6 @@
DuckDuckGo Weather
~~~~~~~~~~~~~~~~~~
"""
from __future__ import annotations
import typing as t
from json import loads
@@ -13,19 +12,11 @@ from dateutil import parser as date_parser
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
from searx.engines.duckduckgo import get_ddg_lang
from searx.enginelib.traits import EngineTraits
from searx.result_types import EngineResults
from searx.extended_types import SXNG_Response
from searx import weather
if t.TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
"website": 'https://duckduckgo.com/',

7
searx/engines/flickr_noapi.py
View File

@@ -3,19 +3,12 @@
"""
from typing import TYPE_CHECKING
import json
from time import time
import re
from urllib.parse import urlencode
from searx.utils import ecma_unescape, html_to_text
if TYPE_CHECKING:
import logging
logger: logging.Logger
# about
about = {
"website": 'https://www.flickr.com',

10
searx/engines/google.py
View File

@@ -10,9 +10,6 @@ engines:
- :ref:`google autocomplete`
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import re
import random
@@ -31,13 +28,6 @@ from searx.exceptions import SearxEngineCaptchaException
from searx.enginelib.traits import EngineTraits
from searx.result_types import EngineResults
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {

10
searx/engines/google_images.py
View File

@@ -13,8 +13,6 @@ This internal API offer results in
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from json import loads
@@ -25,14 +23,6 @@ from searx.engines.google import (
detect_google_sorry,
)
if TYPE_CHECKING:
import logging
from searx.enginelib.traits import EngineTraits
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://images.google.com',

9
searx/engines/google_news.py
View File

@@ -24,8 +24,6 @@ The google news API ignores some parameters from the common :ref:`google API`:
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode
import base64
from lxml import html
@@ -46,13 +44,6 @@ from searx.engines.google import (
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://news.google.com',

13
searx/engines/google_scholar.py
View File

@@ -7,9 +7,6 @@ can make use of the :ref:`google API` to assemble the arguments of the GET
request.
"""
from typing import TYPE_CHECKING
from typing import Optional
from urllib.parse import urlencode
from datetime import datetime
from lxml import html
@@ -28,14 +25,6 @@ from searx.engines.google import (
get_google_info,
time_range_dict,
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
@@ -115,7 +104,7 @@ def request(query, params):
return params
def parse_gs_a(text: Optional[str]):
def parse_gs_a(text: str | None):
"""Parse the text written in green.
Possible formats:

3
searx/engines/google_videos.py
View File

@@ -32,11 +32,8 @@ from searx.engines.google import (
ui_async,
parse_data_images,
)
from searx.enginelib.traits import EngineTraits
from searx.utils import get_embeded_stream_url
traits: EngineTraits
# about
about = {
"website": 'https://www.google.com',

8
searx/engines/mariadb_server.py
View File

@@ -26,8 +26,6 @@ Implementations
"""
from typing import TYPE_CHECKING
try:
import mariadb # pyright: ignore [reportMissingImports]
except ImportError:
@@ -37,12 +35,6 @@ except ImportError:
from searx.result_types import EngineResults
if TYPE_CHECKING:
import logging
logger = logging.getLogger()
engine_type = 'offline'
host = "127.0.0.1"

10
searx/engines/mediawiki.py
View File

@@ -32,21 +32,11 @@ Implementations
===============
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from datetime import datetime
from urllib.parse import urlencode, quote
from searx.utils import html_to_text
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {

9
searx/engines/mojeek.py
View File

@@ -1,8 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Mojeek (general, images, news)"""
from typing import TYPE_CHECKING
from datetime import datetime
from urllib.parse import urlencode
from lxml import html
@@ -50,13 +48,6 @@ region_param = 'arc'
_delta_kwargs = {'day': 'days', 'week': 'weeks', 'month': 'months', 'year': 'years'}
if TYPE_CHECKING:
import logging
logger = logging.getLogger()
traits: EngineTraits
def init(_):
if search_type not in ('', 'images', 'news'):

19
searx/engines/mullvad_leta.py
View File

@@ -36,10 +36,8 @@ Implementations
===============
"""
import typing as t
from __future__ import annotations
import typing
from urllib.parse import urlencode
import babel
from httpx import Response
@@ -49,13 +47,6 @@ from searx.locales import get_official_locales, language_tag, region_tag
from searx.utils import eval_xpath_list
from searx.result_types import EngineResults, MainResult
if typing.TYPE_CHECKING:
import logging
logger = logging.getLogger()
traits: EngineTraits
search_url = "https://leta.mullvad.net"
# about
@@ -80,7 +71,7 @@ time_range_dict = {
"year": "y",
}
LetaEnginesType = typing.Literal["google", "brave"]
LetaEnginesType = t.Literal["google", "brave"]
"""Engine types supported by mullvadleta."""
leta_engine: LetaEnginesType = "google"
@@ -88,12 +79,12 @@ leta_engine: LetaEnginesType = "google"
def init(_):
l = typing.get_args(LetaEnginesType)
l = t.get_args(LetaEnginesType)
if leta_engine not in l:
raise ValueError(f"leta_engine '{leta_engine}' is invalid, use one of {', '.join(l)}")
class DataNodeQueryMetaDataIndices(typing.TypedDict):
class DataNodeQueryMetaDataIndices(t.TypedDict):
"""Indices into query metadata."""
success: int
@@ -112,7 +103,7 @@ class DataNodeQueryMetaDataIndices(typing.TypedDict):
previous: int
class DataNodeResultIndices(typing.TypedDict):
class DataNodeResultIndices(t.TypedDict):
"""Indices into query resultsdata."""
link: int

2
searx/engines/odysee.py
View File

@@ -14,8 +14,6 @@ from searx.network import get
from searx.locales import language_tag
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
# Engine metadata
about = {
"website": "https://odysee.com/",

2
searx/engines/peertube.py
View File

@@ -17,8 +17,6 @@ from searx.locales import language_tag
from searx.utils import html_to_text, humanize_number
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
about = {
# pylint: disable=line-too-long
"website": 'https://joinpeertube.org',

2
searx/engines/qwant.py
View File

@@ -64,8 +64,6 @@ from searx.utils import (
get_embeded_stream_url,
)
traits: EngineTraits
# about
about = {
"website": 'https://www.qwant.com/',

9
searx/engines/radio_browser.py
View File

@@ -5,9 +5,6 @@
https://de1.api.radio-browser.info/#Advanced_station_search
"""
from __future__ import annotations
import typing
import random
import socket
from urllib.parse import urlencode
@@ -19,12 +16,6 @@ from searx.enginelib import EngineCache
from searx.enginelib.traits import EngineTraits
from searx.locales import language_tag
if typing.TYPE_CHECKING:
import logging
logger = logging.getLogger()
traits: EngineTraits
about = {
"website": 'https://www.radio-browser.info/',

10
searx/engines/senscritique.py
View File

@@ -1,10 +1,10 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""SensCritique (movies)
"""
from __future__ import annotations
import typing as t
from json import dumps, loads
from typing import Any, Optional
from searx.result_types import EngineResults, MainResult
about = {
@@ -61,7 +61,7 @@ graphql_query = """query SearchProductExplorer($query: String, $offset: Int, $li
}"""
def request(query: str, params: dict[str, Any]) -> dict[str, Any]:
def request(query: str, params: dict[str, t.Any]) -> dict[str, t.Any]:
offset = (params['pageno'] - 1) * page_size
data = {
@@ -95,7 +95,7 @@ def response(resp) -> EngineResults:
return res
def parse_item(item: dict[str, Any]) -> MainResult | None:
def parse_item(item: dict[str, t.Any]) -> MainResult | None:
"""Parse a single item from the SensCritique API response"""
title = item.get('title', '')
if not title:
@@ -118,7 +118,7 @@ def parse_item(item: dict[str, Any]) -> MainResult | None:
)
def build_content_parts(item: dict[str, Any], title: str, original_title: Optional[str]) -> list[str]:
def build_content_parts(item: dict[str, t.Any], title: str, original_title: str | None) -> list[str]:
"""Build the content parts for an item"""
content_parts = []

10
searx/engines/sepiasearch.py
View File

@@ -5,8 +5,6 @@ peertube engines.
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from datetime import datetime
@@ -17,14 +15,6 @@ from searx.engines.peertube import (
safesearch_table,
time_range_table,
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
about = {
# pylint: disable=line-too-long

6
searx/engines/soundcloud.py
View File

@@ -3,7 +3,6 @@
from __future__ import annotations
import re
import typing
import datetime
from urllib.parse import quote_plus, urlencode
@@ -14,11 +13,6 @@ from lxml import html
from searx.network import get as http_get
from searx.enginelib import EngineCache
if typing.TYPE_CHECKING:
import logging
logger: logging.Logger
about = {
"website": "https://soundcloud.com",
"wikidata_id": "Q568769",

4
searx/engines/sqlite.py
View File

@@ -44,7 +44,7 @@ Implementations
===============
"""
import typing
import typing as t
import sqlite3
import contextlib
@@ -59,7 +59,7 @@ database = ""
query_str = ""
"""SQL query that returns the result items."""
result_type: typing.Literal["MainResult", "KeyValue"] = "KeyValue"
result_type: t.Literal["MainResult", "KeyValue"] = "KeyValue"
"""The result type can be :py:obj:`MainResult` or :py:obj:`KeyValue`."""
limit = 10

13
searx/engines/startpage.py
View File

@@ -78,9 +78,9 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
"""
# pylint: disable=too-many-statements
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import typing as t
from collections import OrderedDict
import re
from unicodedata import normalize, combining
@@ -98,13 +98,6 @@ from searx.locales import region_tag
from searx.enginelib.traits import EngineTraits
from searx.enginelib import EngineCache
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://startpage.com',
@@ -377,7 +370,7 @@ def _get_news_result(result):
}
def _get_image_result(result) -> dict[str, Any] | None:
def _get_image_result(result) -> dict[str, t.Any] | None:
url = result.get('altClickUrl')
if not url:
return None

2
searx/engines/stract.py
View File

@@ -22,8 +22,6 @@ paging = True
base_url = "https://stract.com/beta/api"
search_url = base_url + "/search"
traits: EngineTraits
def request(query, params):
params['url'] = search_url

6
searx/engines/tagesschau.py
View File

@@ -15,17 +15,11 @@ This SearXNG engine uses the `/api2u/search`_ API.
.. _OpenAPI: https://swagger.io/specification/
"""
from typing import TYPE_CHECKING
from datetime import datetime
from urllib.parse import urlencode
import re
if TYPE_CHECKING:
import logging
logger: logging.Logger
about = {
'website': "https://tagesschau.de",
'wikidata_id': "Q703907",

6
searx/engines/tineye.py
View File

@@ -14,18 +14,12 @@ billion images `[tineye.com] <https://tineye.com/how>`_.
"""
from typing import TYPE_CHECKING
from urllib.parse import urlencode
from datetime import datetime
from flask_babel import gettext
from searx.result_types import EngineResults
if TYPE_CHECKING:
import logging
logger = logging.getLogger()
about = {
"website": 'https://tineye.com',
"wikidata_id": 'Q2382535',

26
searx/engines/torznab.py
View File

@@ -47,10 +47,8 @@ Implementations
===============
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import List, Dict, Any
import typing as t
from datetime import datetime
from urllib.parse import quote
from lxml import etree # type: ignore
@@ -58,14 +56,12 @@ from lxml import etree # type: ignore
from searx.exceptions import SearxEngineAPIException
from searx.utils import humanize_bytes
if TYPE_CHECKING:
import httpx
import logging
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
logger: logging.Logger
# engine settings
about: Dict[str, Any] = {
about: dict[str, t.Any] = {
"website": None,
"wikidata_id": None,
"official_api_documentation": "https://torznab.github.io/spec-1.3-draft",
@@ -73,7 +69,7 @@ about: Dict[str, Any] = {
"require_api_key": False,
"results": 'XML',
}
categories: List[str] = ['files']
categories: list[str] = ['files']
paging: bool = False
time_range_support: bool = False
@@ -82,7 +78,7 @@ time_range_support: bool = False
base_url: str = ''
api_key: str = ''
# https://newznab.readthedocs.io/en/latest/misc/api/#predefined-categories
torznab_categories: List[str] = []
torznab_categories: list[str] = []
show_torrent_files: bool = False
show_magnet_links: bool = True
@@ -93,7 +89,7 @@ def init(engine_settings=None): # pylint: disable=unused-argument
raise ValueError('missing torznab base_url')
def request(query: str, params: Dict[str, Any]) -> Dict[str, Any]:
def request(query: str, params: dict[str, t.Any]) -> dict[str, t.Any]:
"""Build the request params."""
search_url: str = base_url + '?t=search&q={search_query}'
@@ -109,7 +105,7 @@ def request(query: str, params: Dict[str, Any]) -> Dict[str, Any]:
return params
def response(resp: httpx.Response) -> List[Dict[str, Any]]:
def response(resp: "SXNG_Response") -> list[dict[str, t.Any]]:
"""Parse the XML response and return a list of results."""
results = []
search_results = etree.XML(resp.content)
@@ -122,13 +118,13 @@ def response(resp: httpx.Response) -> List[Dict[str, Any]]:
item: etree.Element
for item in channel.iterfind('item'):
result: Dict[str, Any] = build_result(item)
result: dict[str, t.Any] = build_result(item)
results.append(result)
return results
def build_result(item: etree.Element) -> Dict[str, Any]:
def build_result(item: etree.Element) -> dict[str, t.Any]:
"""Build a result from a XML item."""
# extract attributes from XML
@@ -150,7 +146,7 @@ def build_result(item: etree.Element) -> Dict[str, Any]:
peers = get_torznab_attribute(item, 'peers')
# map attributes to SearXNG result
result: Dict[str, Any] = {
result: dict[str, t.Any] = {
'template': 'torrent.html',
'title': get_attribute(item, 'title'),
'filesize': humanize_bytes(int(filesize)) if filesize else None,

8
searx/engines/wikidata.py
View File

@@ -5,7 +5,6 @@ from :ref:`wikipedia engine`.
"""
# pylint: disable=missing-class-docstring
from typing import TYPE_CHECKING
from hashlib import md5
from urllib.parse import urlencode, unquote
from json import loads
@@ -23,13 +22,6 @@ from searx.engines.wikipedia import (
)
from searx.enginelib.traits import EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
traits: EngineTraits
# about
about = {
"website": 'https://wikidata.org/',

2
searx/engines/wikipedia.py
View File

@@ -64,8 +64,6 @@ from searx import network as _network
from searx import locales
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
# about
about = {
"website": 'https://www.wikipedia.org/',

9
searx/engines/yahoo.py
View File

@@ -6,7 +6,6 @@ found in :py:obj:`lang2domain` URL ``<lang>.search.yahoo.com`` is used.
"""
from typing import TYPE_CHECKING
from urllib.parse import (
unquote,
urlencode,
@@ -19,14 +18,6 @@ from searx.utils import (
extract_text,
html_to_text,
)
from searx.enginelib.traits import EngineTraits
traits: EngineTraits
if TYPE_CHECKING:
import logging
logger: logging.Logger
# about
about = {

38
searx/engines/zlibrary.py
View File

@@ -32,27 +32,23 @@ Implementations
===============
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import List, Dict, Any, Optional
import typing as t
from datetime import datetime
from urllib.parse import quote
from lxml import html
from flask_babel import gettext
from flask_babel import gettext # pyright: ignore[reportUnknownVariableType]
from searx.utils import extract_text, eval_xpath, eval_xpath_list
from searx.enginelib.traits import EngineTraits
from searx.data import ENGINE_TRAITS
from searx.exceptions import SearxException
if TYPE_CHECKING:
import httpx
import logging
logger: logging.Logger
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
# about
about: Dict[str, Any] = {
about: dict[str, t.Any] = {
"website": "https://zlibrary-global.se",
"wikidata_id": "Q104863992",
"official_api_documentation": None,
@@ -61,7 +57,7 @@ about: Dict[str, Any] = {
"results": "HTML",
}
categories: List[str] = ["files"]
categories: list[str] = ["files"]
paging: bool = True
base_url: str = "https://zlibrary-global.se"
@@ -79,7 +75,7 @@ zlib_ext: str = ""
"""
def init(engine_settings=None) -> None: # pylint: disable=unused-argument
def init(engine_settings: dict[str, t.Any] | None = None) -> None: # pylint: disable=unused-argument
"""Check of engine's settings."""
traits: EngineTraits = EngineTraits(**ENGINE_TRAITS["z-library"])
@@ -91,7 +87,7 @@ def init(engine_settings=None) -> None: # pylint: disable=unused-argument
raise ValueError(f"invalid setting year_to: {zlib_year_to}")
def request(query: str, params: Dict[str, Any]) -> Dict[str, Any]:
def request(query: str, params: dict[str, t.Any]) -> dict[str, t.Any]:
lang: str = traits.get_language(params["language"], traits.all_locale) # type: ignore
search_url: str = (
base_url
@@ -117,8 +113,8 @@ def domain_is_seized(dom):
return bool(dom.xpath('//title') and "seized" in dom.xpath('//title')[0].text.lower())
def response(resp: httpx.Response) -> List[Dict[str, Any]]:
results: List[Dict[str, Any]] = []
def response(resp: "SXNG_Response") -> list[dict[str, t.Any]]:
results: list[dict[str, t.Any]] = []
dom = html.fromstring(resp.text)
if domain_is_seized(dom):
@@ -139,7 +135,7 @@ i18n_book_rating = gettext("Book rating")
i18n_file_quality = gettext("File quality")
def _parse_result(item) -> Dict[str, Any]:
def _parse_result(item) -> dict[str, t.Any]:
author_elements = eval_xpath_list(item, './/div[@class="authors"]//a[@itemprop="author"]')
@@ -152,7 +148,7 @@ def _parse_result(item) -> Dict[str, Any]:
"type": _text(item, './/div[contains(@class, "property__file")]//div[contains(@class, "property_value")]'),
}
thumbnail = _text(item, './/img[contains(@class, "cover")]/@data-src')
thumbnail: str = _text(item, './/img[contains(@class, "cover")]/@data-src')
if not thumbnail.startswith('/'):
result["thumbnail"] = thumbnail
@@ -199,7 +195,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None:
_use_old_values()
return
if not resp.ok: # type: ignore
if not resp.ok:
raise RuntimeError("Response from zlibrary's search page is not OK.")
dom = html.fromstring(resp.text) # type: ignore
@@ -220,20 +216,20 @@ def fetch_traits(engine_traits: EngineTraits) -> None:
engine_traits.custom["year_to"].append(year.get("value"))
for ext in eval_xpath_list(dom, "//div[@id='advSearch-noJS']//select[@id='sf_extensions']/option"):
value: Optional[str] = ext.get("value")
value: str | None = ext.get("value")
if value is None:
value = ""
engine_traits.custom["ext"].append(value)
# Handle languages
# Z-library uses English names for languages, so we need to map them to their respective locales
language_name_locale_map: Dict[str, babel.Locale] = {}
language_name_locale_map: dict[str, babel.Locale] = {}
for locale in babel.core.localedata.locale_identifiers(): # type: ignore
# Create a Locale object for the current locale
loc = babel.Locale.parse(locale)
if loc.english_name is None:
continue
language_name_locale_map[loc.english_name.lower()] = loc # type: ignore
language_name_locale_map[loc.english_name.lower()] = loc
for x in eval_xpath_list(dom, "//div[@id='advSearch-noJS']//select[@id='sf_languages']/option"):
eng_lang = x.get("value")