[feat] metrics: support for open metrics

docs/admin/settings/settings_general.rst

@@ -13,6 +13,7 @@
      donation_url: false
      contact_url: false
      enable_metrics: true
+     open_metrics: ''
 
 ``debug`` : ``$SEARXNG_DEBUG``
   Allow a more detailed log if you run SearXNG directly. Display *detailed* error
@@ -32,3 +33,10 @@
 ``enable_metrics``:
   Enabled by default. Record various anonymous metrics available at ``/stats``,
   ``/stats/errors`` and ``/preferences``.
+
+``open_metrics``:
+  Disabled by default. Set to a secret password to expose an
+  `OpenMetrics API <https://github.com/prometheus/OpenMetrics>`_ at ``/metrics``,
+  e.g. for usage with Prometheus. The ``/metrics`` endpoint is using HTTP Basic Auth,
+  where the password is the value of ``open_metrics`` set above. The username used for
+  Basic Auth can be randomly chosen as only the password is being validated.

searx/metrics/__init__.py

@@ -8,6 +8,7 @@ from timeit import default_timer
 from operator import itemgetter
 
 from searx.engines import engines
+from searx.openmetrics import OpenMetricsFamily
 from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage
 from .error_recorder import count_error, count_exception, errors_per_engines
 
@@ -149,7 +150,9 @@ def get_reliabilities(engline_name_list, checker_results):
         checker_result = checker_results.get(engine_name, {})
         checker_success = checker_result.get('success', True)
         errors = engine_errors.get(engine_name) or []
-        if counter('engine', engine_name, 'search', 'count', 'sent') == 0:
+        sent_count = counter('engine', engine_name, 'search', 'count', 'sent')
+
+        if sent_count == 0:
             # no request
             reliability = None
         elif checker_success and not errors:
@@ -164,8 +167,9 @@ def get_reliabilities(engline_name_list, checker_results):
 
         reliabilities[engine_name] = {
             'reliability': reliability,
+            'sent_count': sent_count,
             'errors': errors,
-            'checker': checker_results.get(engine_name, {}).get('errors', {}),
+            'checker': checker_result.get('errors', {}),
         }
     return reliabilities
 
@@ -245,3 +249,53 @@ def get_engines_stats(engine_name_list):
         'max_time': math.ceil(max_time_total or 0),
         'max_result_count': math.ceil(max_result_count or 0),
     }
+
+
+def openmetrics(engine_stats, engine_reliabilities):
+    metrics = [
+        OpenMetricsFamily(
+            key="searxng_engines_response_time_total_seconds",
+            type_hint="gauge",
+            help_hint="The average total response time of the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine['total'] for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_response_time_processing_seconds",
+            type_hint="gauge",
+            help_hint="The average processing response time of the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine['processing'] for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_response_time_http_seconds",
+            type_hint="gauge",
+            help_hint="The average HTTP response time of the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine['http'] for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_result_count_total",
+            type_hint="counter",
+            help_hint="The total amount of results returned by the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine['result_count'] for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_request_count_total",
+            type_hint="counter",
+            help_hint="The total amount of user requests made to this engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[engine_reliabilities.get(engine['name'], {}).get('sent_count', 0) for engine in engine_stats['time']],
+        ),
+        OpenMetricsFamily(
+            key="searxng_engines_reliability_total",
+            type_hint="counter",
+            help_hint="The overall reliability of the engine",
+            data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']],
+            data=[
+                engine_reliabilities.get(engine['name'], {}).get('reliability', 0) for engine in engine_stats['time']
+            ],
+        ),
+    ]
+    return "".join([str(metric) for metric in metrics])

searx/openmetrics.py

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Module providing support for displaying data in OpenMetrics format"""
+
+
+class OpenMetricsFamily:  # pylint: disable=too-few-public-methods
+    """A family of metrics.
+    The key parameter is the metric name that should be used (snake case).
+    The type_hint parameter must be one of 'counter', 'gauge', 'histogram', 'summary'.
+    The help_hint parameter is a short string explaining the metric.
+    The data_info parameter is a dictionary of descriptionary parameters for the data point (e.g. request method/path).
+    The data parameter is a flat list of the actual data in shape of a primive type.
+
+    See https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md for more information.
+    """
+
+    def __init__(self, key: str, type_hint: str, help_hint: str, data_info: list, data: list):
+        self.key = key
+        self.type_hint = type_hint
+        self.help_hint = help_hint
+        self.data_info = data_info
+        self.data = data
+
+    def __str__(self):
+        text_representation = f"""# HELP {self.key} {self.help_hint}
+# TYPE {self.key} {self.type_hint}
+"""
+
+        for i, data_info_dict in enumerate(self.data_info):
+            if not data_info_dict and data_info_dict != 0:
+                continue
+
+            info_representation = ','.join([f"{key}=\"{value}\"" for (key, value) in data_info_dict.items()])
+            text_representation += f"{self.key}{{{info_representation}}} {self.data[i]}\n"
+
+        return text_representation

searx/settings.yml

@@ -12,6 +12,10 @@ general:
   contact_url: false
   # record stats
   enable_metrics: true
+  # expose stats in open metrics format at /metrics
+  # leave empty to disable (no password set)
+  # open_metrics: <password>
+  open_metrics: ''
 
 brand:
   new_issue_url: https://github.com/searxng/searxng/issues/new

searx/settings_defaults.py

@@ -143,6 +143,7 @@ SCHEMA = {
         'contact_url': SettingsValue((None, False, str), None),
         'donation_url': SettingsValue((bool, str), "https://docs.searxng.org/donate.html"),
         'enable_metrics': SettingsValue(bool, True),
+        'open_metrics': SettingsValue(str, ''),
     },
     'brand': {
         'issue_url': SettingsValue(str, 'https://github.com/searxng/searxng/issues'),

searx/webapp.py

@@ -87,10 +87,7 @@ from searx.webadapter import (
     get_selected_categories,
     parse_lang,
 )
-from searx.utils import (
+from searx.utils import gen_useragent, dict_subset
-    gen_useragent,
-    dict_subset,
-)
 from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH
 from searx.query import RawTextQuery
 from searx.plugins import Plugin, plugins, initialize as plugin_initialize
@@ -104,13 +101,7 @@ from searx.answerers import (
     answerers,
     ask,
 )
-from searx.metrics import (
+from searx.metrics import get_engines_stats, get_engine_errors, get_reliabilities, histogram, counter, openmetrics
-    get_engines_stats,
-    get_engine_errors,
-    get_reliabilities,
-    histogram,
-    counter,
-)
 from searx.flaskfix import patch_application
 
 from searx.locales import (
@@ -1210,6 +1201,30 @@ def stats_checker():
     return jsonify(result)
 
 
+@app.route('/metrics')
+def stats_open_metrics():
+    password = settings['general'].get("open_metrics")
+
+    if not (settings['general'].get("enable_metrics") and password):
+        return Response('open metrics is disabled', status=404, mimetype='text/plain')
+
+    if not request.authorization or request.authorization.password != password:
+        return Response('access forbidden', status=401, mimetype='text/plain')
+
+    filtered_engines = dict(filter(lambda kv: request.preferences.validate_token(kv[1]), engines.items()))
+
+    checker_results = checker_get_result()
+    checker_results = (
+        checker_results['engines'] if checker_results['status'] == 'ok' and 'engines' in checker_results else {}
+    )
+
+    engine_stats = get_engines_stats(filtered_engines)
+    engine_reliabilities = get_reliabilities(filtered_engines, checker_results)
+    metrics_text = openmetrics(engine_stats, engine_reliabilities)
+
+    return Response(metrics_text, mimetype='text/plain')
+
+
 @app.route('/robots.txt', methods=['GET'])
 def robots():
     return Response(