[doc] Development Quickstart: documentation of the tools (dev.env)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

docs/build-templates/searxng.rst

@@ -169,7 +169,7 @@ ${fedora_build}
        $ sudo -H -u ${SERVICE_USER} -i
        (${SERVICE_USER})$ cd ${SEARXNG_SRC}
        (${SERVICE_USER})$ export SEARXNG_SETTINGS_PATH=\"${SEARXNG_SETTINGS_PATH}\"
-       (${SERVICE_USER})$ python searx/webapp.py
+       (${SERVICE_USER})$ python -m searx.webapp
 
        # disable debug
        $ sudo -H sed -i -e \"s/debug : True/debug : False/g\" \"$SEARXNG_SETTINGS_PATH\"

docs/dev/quickstart.rst

@@ -60,11 +60,29 @@ into the developer environment and start a python based HTTP server by::
 
     $ ./manage dev.env
     ...
-    (dev.env)$ SEARXNG_DEBUG=1 python -m searx.webapp
+    (dev.env)$ SEARXNG_DEBUG=1 searxng-run
 
 Since this is a pure Python solution, you can set breakpoints in your code with
 ``pdb.set_trace()`` and the debugger will wait for you in the terminal prompt.
 
+Any other script or command line provided by SearXNG can also be used in the
+same environment, here are a few examples::
+
+    # tools related to favicons
+    (dev.env)$ python -m searx.favicons
+
+    # tools related to DATA stored in searx/data
+    (dev.env)$ python -m searx.data --help
+
+    # tools related to engines
+    (dev.env)$ python -m searx.enginelib --help
+
+    # to test one of the update scripts
+    (dev.env)$ searxng_extra/update/update_engine_traits.py --help
+
+    # to test the update of the wikidata units
+    (dev.env)$ searxng_extra/update/update_wikidata_units.py
+
 
 .. sidebar:: further read
 

searxng_extra/update/update_engine_traits.py

@@ -1,23 +1,26 @@
 #!/usr/bin/env python
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Update :py:obj:`searx.enginelib.traits.EngineTraitsMap` and :origin:`searx/languages.py`
+"""Update :py:obj:`searx.enginelib.traits.EngineTraitsMap` and :origin:`searx/sxng_locales.py`
 
 :py:obj:`searx.enginelib.traits.EngineTraitsMap.ENGINE_TRAITS_FILE`:
   Persistence of engines traits, fetched from the engines.
 
-:origin:`searx/languages.py`
+:origin:`searx/sxng_locales.py`
   Is generated  from intersecting each engine's supported traits.
 
 The script :origin:`searxng_extra/update/update_engine_traits.py` is called in
 the :origin:`CI Update data ... <.github/workflows/data-update.yml>`
 
 """
-
 # pylint: disable=invalid-name
+
+import typing as t
+
 from unicodedata import lookup
 from pathlib import Path
 from pprint import pformat
 import babel
+import typer
 
 from searx import settings, searx_dir
 from searx import network
@@ -25,8 +28,8 @@ from searx.engines import load_engines
 from searx.enginelib.traits import EngineTraitsMap
 
 # Output files.
-languages_file = Path(searx_dir) / 'sxng_locales.py'
-languages_file_header = """\
+sxng_locales_file = Path(searx_dir) / 'sxng_locales.py'
+sxng_locales_file_header = """\
 # SPDX-License-Identifier: AGPL-3.0-or-later
 '''List of SearXNG's locale codes used for the search language/region.
 
@@ -39,7 +42,7 @@ languages_file_header = """\
 
 sxng_locales = (
 """
-languages_file_footer = """,
+sxng_locales_file_footer = """,
 )
 '''
 A list of five-digit tuples:
@@ -75,43 +78,63 @@ lang2emoji = {
     'he': '\U0001f1ee\U0001f1f1',  # Hebrew
 }
 
+app = typer.Typer()
 
-def main():
 
-    engines_cfg = []
+@app.command()
+def cli(engines: t.Annotated[list[str] | None, typer.Argument()] = None):
+    """Update ``data/engine_traits.json`` and ``sxng_locales.py``.
 
+    Optionally, if arguments are provided via the command line, these are
+    interpreted as the names of the engines that should be updated.  All other
+    engines will be left untouched.
+    """
+
+    all_eng_names: list[str] = [e["name"] for e in settings["engines"]]
+    if engines:
+        unknown: list[str] = [e for e in engines if e not in all_eng_names]
+        if unknown:
+            print(f"ERROR: unknown engines --> {', '.join(unknown)}")
+            raise typer.Exit(42)
+
+    engines_cfg: list[dict[str, t.Any]] = []
     for eng_data in settings["engines"]:
-        eng_data["inactive"] = False
-        engines_cfg.append(eng_data)
+        if not engines or eng_data["name"] in engines:
+            eng_data["inactive"] = False
+            engines_cfg.append(eng_data)
 
     load_engines(engines_cfg)
-    # traits_map = EngineTraitsMap.from_data()
-    traits_map = fetch_traits_map()
+    traits_map: EngineTraitsMap = fetch_traits_map()
+    if engines:
+        _map = EngineTraitsMap.from_data()
+        _map.update(traits_map)
+        traits_map = _map
+
+    print("write json file: %s" % traits_map.ENGINE_TRAITS_FILE)
+    traits_map.save_data()
     sxng_tag_list = filter_locales(traits_map)
-    write_languages_file(sxng_tag_list)
+    write_sxng_locales_file(sxng_tag_list)
 
 
-def fetch_traits_map():
+def fetch_traits_map() -> EngineTraitsMap:
     """Fetches supported languages for each engine and writes json file with those."""
     network.set_timeout_for_thread(10.0)
 
-    def log(msg):
+    def log(msg: str):
         print(msg)
 
     traits_map = EngineTraitsMap.fetch_traits(log=log)
     print("fetched properties from %s engines" % len(traits_map))
-    print("write json file: %s" % traits_map.ENGINE_TRAITS_FILE)
-    traits_map.save_data()
     return traits_map
 
 
-def filter_locales(traits_map: EngineTraitsMap):
+def filter_locales(traits_map: EngineTraitsMap) -> set[str]:
     """Filter language & region tags by a threshold."""
 
     min_eng_per_region = 18
     min_eng_per_lang = 22
 
-    _ = {}
+    _: dict[str, int] = {}
     for eng in traits_map.values():
         for reg in eng.regions.keys():
             _[reg] = _.get(reg, 0) + 1
@@ -131,7 +154,7 @@ def filter_locales(traits_map: EngineTraitsMap):
 
     languages = set(k for k, v in _.items() if v >= min_eng_per_lang)
 
-    sxng_tag_list = set()
+    sxng_tag_list: set[str] = set()
     sxng_tag_list.update(regions)
     sxng_tag_list.update(lang_from_region)
     sxng_tag_list.update(languages)
@@ -139,9 +162,9 @@ def filter_locales(traits_map: EngineTraitsMap):
     return sxng_tag_list
 
 
-def write_languages_file(sxng_tag_list):
+def write_sxng_locales_file(sxng_tag_list: set[str]):
 
-    language_codes = []
+    language_codes: list[tuple[str, str, str, str, str]] = []
 
     for sxng_tag in sorted(sxng_tag_list):
         sxng_locale: babel.Locale = babel.Locale.parse(sxng_tag, sep='-')
@@ -150,7 +173,7 @@ def write_languages_file(sxng_tag_list):
 
         item = (
             sxng_tag,
-            sxng_locale.get_language_name().title(),  # type: ignore
+            sxng_locale.get_language_name().title(),  # pyright: ignore[reportOptionalMemberAccess]
             sxng_locale.get_territory_name() or '',
             sxng_locale.english_name.split(' (')[0] if sxng_locale.english_name else '',
             UnicodeEscape(flag),
@@ -158,13 +181,13 @@ def write_languages_file(sxng_tag_list):
 
         language_codes.append(item)
 
-    language_codes = tuple(language_codes)
+    _codes = tuple(language_codes)
 
-    with languages_file.open('w', encoding='utf-8') as new_file:
+    with sxng_locales_file.open('w', encoding='utf-8') as new_file:
         file_content = "{header} {language_codes}{footer}".format(
-            header=languages_file_header,
-            language_codes=pformat(language_codes, width=120, indent=4)[1:-1],
-            footer=languages_file_footer,
+            header=sxng_locales_file_header,
+            language_codes=pformat(_codes, width=120, indent=4)[1:-1],
+            footer=sxng_locales_file_footer,
         )
         new_file.write(file_content)
         new_file.close()
@@ -203,4 +226,4 @@ def get_unicode_flag(locale: babel.Locale):
 
 
 if __name__ == "__main__":
-    main()
+    app()

utils/searxng.sh

@@ -661,7 +661,7 @@ searxng.instance.localtest() {
     tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"
 export SEARXNG_SETTINGS_PATH="${SEARXNG_SETTINGS_PATH}"
 cd ${SEARXNG_SRC}
-timeout 10 python searx/webapp.py &
+timeout 10 python -m searx/webapp &
 sleep 3
 curl --location --verbose --head --insecure ${SEARXNG_INTERNAL_HTTP}
 EOF