Merge 443fcc7233 into 2fbf15eccb

Bumps [typer-slim](https://github.com/fastapi/typer) from 0.12.5 to 0.13.0.
- [Release notes](https://github.com/fastapi/typer/releases)
- [Changelog](https://github.com/fastapi/typer/blob/master/docs/release-notes.md)
- [Commits](https://github.com/fastapi/typer/compare/0.12.5...0.13.0)

---
updated-dependencies:
- dependency-name: typer-slim
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

requirements.txt

@@ -18,4 +18,4 @@ fasttext-predict==0.9.2.2
 tomli==2.0.2; python_version < '3.11'
 msgspec==0.18.6
 eval_type_backport; python_version < '3.9'
-typer-slim==0.12.5
+typer-slim==0.13.0

searx/metrics/__init__.py

@@ -8,6 +8,7 @@ from timeit import default_timer
 from operator import itemgetter
 
 from searx.engines import engines
+from searx.openmetrics import OpenMetricsFamily
 from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage
 from .error_recorder import count_error, count_exception, errors_per_engines
 
@@ -149,7 +150,9 @@ def get_reliabilities(engline_name_list, checker_results):
         checker_result = checker_results.get(engine_name, {})
         checker_success = checker_result.get('success', True)
         errors = engine_errors.get(engine_name) or []
-        if counter('engine', engine_name, 'search', 'count', 'sent') == 0:
+        sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
+
+        if sent_count == 0:
             # no request
             reliability = None
         elif checker_success and not errors:
@@ -164,8 +167,9 @@ def get_reliabilities(engline_name_list, checker_results):
 
         reliabilities[engine_name] = {
             'reliability': reliability,
+            'sent_count': sent_count,
             'errors': errors,
-            'checker': checker_results.get(engine_name, {}).get('errors', {}),
+            'checker': checker_result.get('errors', {}),
         }
     return reliabilities
 
@@ -245,3 +249,53 @@ def get_engines_stats(engine_name_list):
         'max_time': math.ceil(max_time_total or 0),
         'max_result_count': math.ceil(max_result_count or 0),
     }
+
+
+def openmetrics(engine_stats, engine_reliabilities):
+    metrics = [
+        OpenMetricsFamily(
+            key="searxng_engines_response_time_total_seconds",
+            type_hint="gauge",
+            help_hint="The average total response time of the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine['total'] for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_response_time_processing_seconds",
+            type_hint="gauge",
+            help_hint="The average processing response time of the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine['processing'] for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_response_time_http_seconds",
+            type_hint="gauge",
+            help_hint="The average HTTP response time of the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine['http'] for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_result_count_total",
+            type_hint="counter",
+            help_hint="The total amount of results returned by the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine['result_count'] for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_request_count_total",
+            type_hint="counter",
+            help_hint="The total amount of user requests made to this engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine_reliabilities.get(engine['name'], {}).get('sent_count', 0) for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_reliability_total",
+            type_hint="counter",
+            help_hint="The overall reliability of the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[
+                engine_reliabilities.get(engine['name'], {}).get('reliability', 0) for engine in engine_stats['time']
+            ],
+        ),
+    ]
+    return "".join([str(metric) for metric in metrics])

searx/openmetrics.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Module providing support for displaying data in OpenMetrics format"""
+
+
+class OpenMetricsFamily:  # pylint: disable=too-few-public-methods
+    """A family of metrics.
+    The key parameter is the metric name that should be used (snake case).
+    The type_hint parameter must be one of 'counter', 'gauge', 'histogram', 'summary'.
+    The help_hint parameter is a short string explaining the metric.
+    The data_info parameter is a dictionary of descriptionary parameters for the data point (e.g. request method/path).
+    The data parameter is a flat list of the actual data in shape of a primive type.
+
+    See https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md for more information.
+    """
+
+    def __init__(self, key: str, type_hint: str, help_hint: str, data_info: list, data: list):
+        self.key = key
+        self.type_hint = type_hint
+        self.help_hint = help_hint
+        self.data_info = data_info
+        self.data = data
+
+    def __str__(self):
+        text_representation = f"""# HELP {self.key} {self.help_hint}
+# TYPE {self.key} {self.type_hint}
+"""
+
+        for i in range(0, len(self.data_info)):
+            if not self.data[i] and self.data[i] != 0:
+                continue
+
+            info_representation = ','.join([f"{key}=\"{value}\"" for (key, value) in self.data_info[i].items()])
+            text_representation += f"{self.key}{{{info_representation}}} {self.data[i]}\n"
+
+        return text_representation

searx/settings.yml

@@ -12,6 +12,10 @@ general:
   contact_url: false
   # record stats
   enable_metrics: true
+  # expose stats in open metrics format at /metrics
+  # leave empty to disable (no password set)
+  # open_metrics: <password>
+  open_metrics: ''
 
 brand:
   new_issue_url: https://github.com/searxng/searxng/issues/new

searx/settings_defaults.py

@@ -143,6 +143,7 @@ SCHEMA = {
         'contact_url': SettingsValue((None, False, str), None),
         'donation_url': SettingsValue((bool, str), "https://docs.searxng.org/donate.html"),
         'enable_metrics': SettingsValue(bool, True),
+        'open_metrics': SettingsValue(str, ''),
     },
     'brand': {
         'issue_url': SettingsValue(str, 'https://github.com/searxng/searxng/issues'),

searx/translations/es/LC_MESSAGES/messages.po

@@ -39,9 +39,8 @@ msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-26 21:13+0000\n"
+"PO-Revision-Date: 2024-11-03 09:08+0000\n"
-"Last-Translator: Atul_Eterno <Atul_Eterno@users.noreply.translate.codeberg."
+"Last-Translator: return42 <return42@users.noreply.translate.codeberg.org>\n"
-"org>\n"
 "Language-Team: Spanish <https://translate.codeberg.org/projects/searxng/"
 "searxng/es/>\n"
 "Language: es\n"
@@ -1246,7 +1245,7 @@ msgstr "Tiempo máximo"
 
 #: searx/templates/simple/preferences/favicon.html:2
 msgid "Favicon Resolver"
-msgstr ""
+msgstr "Buscador de favicon"
 
 #: searx/templates/simple/preferences/favicon.html:15
 msgid "Display favicons near search results"

searx/translations/nb_NO/LC_MESSAGES/messages.po

@@ -10,21 +10,22 @@
 # return42 <return42@users.noreply.translate.codeberg.org>, 2024.
 # omfj <omfj@users.noreply.translate.codeberg.org>, 2024.
 # combwizard <combwizard@users.noreply.translate.codeberg.org>, 2024.
+# laaknor <laaknor@users.noreply.translate.codeberg.org>, 2024.
 msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-04 21:35+0000\n"
+"PO-Revision-Date: 2024-11-03 09:08+0000\n"
-"Last-Translator: combwizard "
+"Last-Translator: laaknor <laaknor@users.noreply.translate.codeberg.org>\n"
-"<combwizard@users.noreply.translate.codeberg.org>\n"
+"Language-Team: Norwegian Bokmål <https://translate.codeberg.org/projects/"
+"searxng/searxng/nb_NO/>\n"
 "Language: nb_NO\n"
-"Language-Team: Norwegian Bokmål "
-"<https://translate.codeberg.org/projects/searxng/searxng/nb_NO/>\n"
-"Plural-Forms: nplurals=2; plural=n != 1;\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -1051,7 +1052,7 @@ msgstr "Det er ingen flere resultater. Du kan prøve å:"
 
 #: searx/templates/simple/messages/no_results.html:19
 msgid "Refresh the page."
-msgstr "oppfrisk siden"
+msgstr "oppfrisk siden."
 
 #: searx/templates/simple/messages/no_results.html:20
 msgid "Search for another query or select another category (above)."
@@ -1882,4 +1883,3 @@ msgstr "skjul video"
 
 #~ msgid "Engines cannot retrieve results"
 #~ msgstr "Søkemotorer kan ikke motta resultater"
-

searx/translations/nl/LC_MESSAGES/messages.po

@@ -23,13 +23,14 @@
 # notlmutsaers <notlmutsaers@users.noreply.translate.codeberg.org>, 2024.
 # return42 <return42@users.noreply.translate.codeberg.org>, 2024.
 # ljansen <ljansen@users.noreply.translate.codeberg.org>, 2024.
+# zarlin <zarlin@users.noreply.translate.codeberg.org>, 2024.
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-28 21:07+0000\n"
+"PO-Revision-Date: 2024-11-02 04:00+0000\n"
-"Last-Translator: ljansen <ljansen@users.noreply.translate.codeberg.org>\n"
+"Last-Translator: zarlin <zarlin@users.noreply.translate.codeberg.org>\n"
 "Language-Team: Dutch <https://translate.codeberg.org/projects/searxng/"
 "searxng/nl/>\n"
 "Language: nl\n"
@@ -493,7 +494,7 @@ msgstr "stemmen"
 
 #: searx/engines/radio_browser.py:107
 msgid "clicks"
-msgstr "clicks"
+msgstr "klikken"
 
 #: searx/engines/seekr.py:193 searx/engines/yummly.py:71
 #: searx/engines/zlibrary.py:137
@@ -662,7 +663,7 @@ msgstr "Voorkeuren"
 
 #: searx/templates/simple/base.html:68
 msgid "Powered by"
-msgstr "Zoekmachine"
+msgstr "Verzorgd door"
 
 #: searx/templates/simple/base.html:68
 msgid "a privacy-respecting, open metasearch engine"
@@ -1069,7 +1070,7 @@ msgstr "Er zijn geen resultaten meer. U kunt proberen om:"
 
 #: searx/templates/simple/messages/no_results.html:19
 msgid "Refresh the page."
-msgstr "Ververs de pagina"
+msgstr "Ververs de pagina."
 
 #: searx/templates/simple/messages/no_results.html:20
 msgid "Search for another query or select another category (above)."
@@ -1235,9 +1236,8 @@ msgid "Max time"
 msgstr "Max. duur"
 
 #: searx/templates/simple/preferences/favicon.html:2
-#, fuzzy
 msgid "Favicon Resolver"
-msgstr "favicon-resolver"
+msgstr "Favicon Oplosser"
 
 #: searx/templates/simple/preferences/favicon.html:15
 msgid "Display favicons near search results"

searx/translations/th/LC_MESSAGES/messages.po

@@ -8,21 +8,23 @@
 # return42 <return42@users.noreply.translate.codeberg.org>, 2024.
 # abhabongse <abhabongse@users.noreply.translate.codeberg.org>, 2024.
 # tutakrab <tutakrab@users.noreply.translate.codeberg.org>, 2024.
+# sahussawud <sahussawud@users.noreply.translate.codeberg.org>, 2024.
 msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-02 16:10+0000\n"
+"PO-Revision-Date: 2024-11-06 07:26+0000\n"
-"Last-Translator: tutakrab <tutakrab@users.noreply.translate.codeberg.org>"
+"Last-Translator: sahussawud <sahussawud@users.noreply.translate.codeberg.org>"
 "\n"
+"Language-Team: Thai <https://translate.codeberg.org/projects/searxng/searxng/"
+"th/>\n"
 "Language: th\n"
-"Language-Team: Thai "
-"<https://translate.codeberg.org/projects/searxng/searxng/th/>\n"
-"Plural-Forms: nplurals=1; plural=0;\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=1; plural=0;\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -163,7 +165,7 @@ msgstr "มืด"
 #. STYLE_NAMES['BLACK']
 #: searx/searxng.msg
 msgid "black"
-msgstr ""
+msgstr "สีดำ"
 
 #. BRAND_CUSTOM_LINKS['UPTIME']
 #: searx/searxng.msg
@@ -331,12 +333,12 @@ msgstr "ผู้เขียน"
 #. SOCIAL_MEDIA_TERMS['THREAD OPEN']
 #: searx/engines/discourse.py:149 searx/searxng.msg
 msgid "open"
-msgstr ""
+msgstr "สร้าง"
 
 #. SOCIAL_MEDIA_TERMS['THREAD CLOSED']
 #: searx/engines/discourse.py:149 searx/searxng.msg
 msgid "closed"
-msgstr ""
+msgstr "ลบ"
 
 #. SOCIAL_MEDIA_TERMS['THREAD ANSWERED']
 #: searx/engines/discourse.py:160 searx/searxng.msg
@@ -450,7 +452,7 @@ msgstr "คำนวณ {functions} จากอาร์กิวเมนต
 
 #: searx/engines/mozhi.py:57
 msgid "Synonyms"
-msgstr ""
+msgstr "คำเหมือน"
 
 #: searx/engines/openstreetmap.py:159
 msgid "Get directions"
@@ -538,8 +540,9 @@ msgid "hash digest"
 msgstr "แฮชย่อย"
 
 #: searx/plugins/hostnames.py:103
+#, fuzzy
 msgid "Hostnames plugin"
-msgstr ""
+msgstr "ชื่อโฮส ปลั๊กอิน"
 
 #: searx/plugins/hostnames.py:104
 msgid "Rewrite hostnames, remove results or prioritize them based on the hostname"
@@ -1698,4 +1701,3 @@ msgstr "ซ่อนวิดีโอ"
 
 #~ msgid "Engines cannot retrieve results"
 #~ msgstr "เครื่องมือไม่สามารถดึงผลลัพธ์ได้"
-

searx/webapp.py

@@ -87,10 +87,7 @@ from searx.webadapter import (
     get_selected_categories,
     parse_lang,
 )
-from searx.utils import (
+from searx.utils import gen_useragent, dict_subset
-    gen_useragent,
-    dict_subset,
-)
 from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
 from searx.query import RawTextQuery
 from searx.plugins import Plugin, plugins, initialize as plugin_initialize
@@ -104,13 +101,7 @@ from searx.answerers import (
     answerers,
     ask,
 )
-from searx.metrics import (
+from searx.metrics import get_engines_stats, get_engine_errors, get_reliabilities, histogram, counter, openmetrics
-    get_engines_stats,
-    get_engine_errors,
-    get_reliabilities,
-    histogram,
-    counter,
-)
 from searx.flaskfix import patch_application
 
 from searx.locales import (
@@ -1218,6 +1209,30 @@ def stats_checker():
     return jsonify(result)
 
 
+@app.route('/metrics')
+def stats_open_metrics():
+    password = settings['general'].get("open_metrics")
+
+    if not (settings['general'].get("enable_metrics") and password):
+        return Response('open metrics is disabled', status=404, mimetype='text/plain')
+
+    if not request.authorization or request.authorization.password != password:
+        return Response('access forbidden', status=401, mimetype='text/plain')
+
+    filtered_engines = dict(filter(lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
+
+    checker_results = checker_get_result()
+    checker_results = (
+        checker_results['engines'] if checker_results['status'] == 'ok' and 'engines' in checker_results else {}
+    )
+
+    engine_stats = get_engines_stats(filtered_engines)
+    engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
+    metrics_text = openmetrics(engine_stats, engine_reliabilities)
+
+    return Response(metrics_text, mimetype='text/plain')
+
+
 @app.route('/robots.txt', methods=['GET'])
 def robots():
     return Response(