[chore] engine preferences: fix grammar mistakes (#4883)

The types necessary for weather information such as GeoLocation, DateTime,
Temperature,Pressure, WindSpeed, RelativeHumidity, Compass (wind direction) and
symbols for the weather have been implemented.

There are unit conversions and translations for weather property labels.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

client/simple/src/less/style.less

@@ -19,6 +19,7 @@
 @import "new_issue.less";
 @import "stats.less";
 @import "result_templates.less";
+@import "weather.less";
 
 // for index.html template
 @import "index.less";

client/simple/src/less/weather.less

@@ -0,0 +1,38 @@
+#answers .weather {
+  summary {
+    display: block;
+    list-style: none;
+  }
+
+  div.summary {
+    margin: 0;
+    padding: 0.5rem 1rem;
+    background-color: var(--color-header-background);
+    .rounded-corners-tiny;
+  }
+
+  table {
+    font-size: 0.9rem;
+    table-layout: fixed;
+    margin-top: 0.5rem;
+    margin-bottom: 0.5rem;
+  }
+
+  td {
+    padding: 0;
+  }
+
+  img.symbol {
+    width: 5rem;
+    margin: auto;
+    display: block;
+  }
+
+  .title {
+    // background-color: var(--color-result-keyvalue-even);
+  }
+
+  .measured {
+    // background-color: var(--color-result-keyvalue-odd);
+  }
+}

docs/src/searx.weather.rst

@@ -0,0 +1,8 @@
+.. _weather:
+
+=======
+Weather
+=======
+
+.. automodule:: searx.weather
+   :members:

searx/babel_extract.py

@@ -45,6 +45,14 @@ def extract(
     namespace = {}
     exec(fileobj.read(), {}, namespace)  # pylint: disable=exec-used
 
-    for name in namespace['__all__']:
-        for k, v in namespace[name].items():
-            yield 0, '_', v, ["%s['%s']" % (name, k)]
+    for obj_name in namespace['__all__']:
+        obj = namespace[obj_name]
+        if isinstance(obj, list):
+            for msg in obj:
+                # (lineno, funcname, message, comments)
+                yield 0, '_', msg, [f"{obj_name}"]
+        elif isinstance(obj, dict):
+            for k, msg in obj.items():
+                yield 0, '_', msg, [f"{obj_name}['{k}']"]
+        else:
+            raise ValueError(f"{obj_name} should be list or dict")

searx/cache.py

@@ -226,7 +226,7 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
     # The key/value tables will be created on demand by self.create_table
     DDL_CREATE_TABLES = {}
 
-    CACHE_TABLE_PREFIX = "CACHE-TABLE-"
+    CACHE_TABLE_PREFIX = "CACHE-TABLE"
 
     def __init__(self, cfg: ExpireCacheCfg):
         """An instance of the SQLite expire cache is build up from a

searx/engines/open_meteo.py

@@ -1,17 +1,17 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Open Meteo (weather)"""
 
-from urllib.parse import urlencode, quote_plus
+from urllib.parse import urlencode
 from datetime import datetime
-from flask_babel import gettext
 
-from searx.network import get
-from searx.exceptions import SearxEngineAPIException
+from searx.result_types import EngineResults, WeatherAnswer
+from searx import weather
+
 
 about = {
-    "website": 'https://open-meteo.com',
+    "website": "https://open-meteo.com",
     "wikidata_id": None,
-    "official_api_documentation": 'https://open-meteo.com/en/docs',
+    "official_api_documentation": "https://open-meteo.com/en/docs",
     "use_official_api": True,
     "require_api_key": False,
     "results": "JSON",
@@ -22,97 +22,130 @@ categories = ["weather"]
 geo_url = "https://geocoding-api.open-meteo.com"
 api_url = "https://api.open-meteo.com"
 
-data_of_interest = "temperature_2m,relative_humidity_2m,apparent_temperature,cloud_cover,pressure_msl,wind_speed_10m,wind_direction_10m"  # pylint: disable=line-too-long
+data_of_interest = (
+    "temperature_2m",
+    "apparent_temperature",
+    "relative_humidity_2m",
+    "apparent_temperature",
+    "cloud_cover",
+    "pressure_msl",
+    "wind_speed_10m",
+    "wind_direction_10m",
+    "weather_code",
+    # "visibility",
+    # "is_day",
+)
 
 
 def request(query, params):
-    location_url = f"{geo_url}/v1/search?name={quote_plus(query)}"
 
-    resp = get(location_url)
-    if resp.status_code != 200:
-        raise SearxEngineAPIException("invalid geo location response code")
+    try:
+        location = weather.GeoLocation.by_query(query)
+    except ValueError:
+        return
 
-    json_locations = resp.json().get("results", [])
-    if len(json_locations) == 0:
-        raise SearxEngineAPIException("location not found")
-
-    location = json_locations[0]
     args = {
-        'latitude': location['latitude'],
-        'longitude': location['longitude'],
-        'timeformat': 'unixtime',
-        'format': 'json',
-        'current': data_of_interest,
-        'forecast_days': 7,
-        'hourly': data_of_interest,
+        "latitude": location.latitude,
+        "longitude": location.longitude,
+        "timeformat": "unixtime",
+        "timezone": "auto",  # use timezone of the location
+        "format": "json",
+        "current": ",".join(data_of_interest),
+        "forecast_days": 3,
+        "hourly": ",".join(data_of_interest),
     }
 
-    params['url'] = f"{api_url}/v1/forecast?{urlencode(args)}"
-
-    return params
+    params["url"] = f"{api_url}/v1/forecast?{urlencode(args)}"
 
 
-def c_to_f(temperature):
-    return "%.2f" % ((temperature * 1.8) + 32)
+# https://open-meteo.com/en/docs#weather_variable_documentation
+# https://nrkno.github.io/yr-weather-symbols/
+
+WMO_TO_CONDITION: dict[int, weather.WeatherConditionType] = {
+    # 0	Clear sky
+    0: "clear sky",
+    # 1, 2, 3     Mainly clear, partly cloudy, and overcast
+    1: "fair",
+    2: "partly cloudy",
+    3: "cloudy",
+    # 45, 48      Fog and depositing rime fog
+    45: "fog",
+    48: "fog",
+    # 51, 53, 55  Drizzle: Light, moderate, and dense intensity
+    51: "light rain",
+    53: "light rain",
+    55: "light rain",
+    # 56, 57      Freezing Drizzle: Light and dense intensity
+    56: "light sleet showers",
+    57: "light sleet",
+    # 61, 63, 65  Rain: Slight, moderate and heavy intensity
+    61: "light rain",
+    63: "rain",
+    65: "heavy rain",
+    # 66, 67    Freezing Rain: Light and heavy intensity
+    66: "light sleet showers",
+    67: "light sleet",
+    # 71, 73, 75  Snow fall: Slight, moderate, and heavy intensity
+    71: "light sleet",
+    73: "sleet",
+    75: "heavy sleet",
+    # 77    Snow grains
+    77: "snow",
+    # 80, 81, 82  Rain showers: Slight, moderate, and violent
+    80: "light rain showers",
+    81: "rain showers",
+    82: "heavy rain showers",
+    # 85, 86      Snow showers slight and heavy
+    85: "snow showers",
+    86: "heavy snow showers",
+    # 95          Thunderstorm: Slight or moderate
+    95: "rain and thunder",
+    # 96, 99      Thunderstorm with slight and heavy hail
+    96: "light snow and thunder",
+    99: "heavy snow and thunder",
+}
 
 
-def get_direction(degrees):
-    if degrees < 45 or degrees >= 315:
-        return "N"
+def _weather_data(location: weather.GeoLocation, data: dict):
 
-    if 45 <= degrees < 135:
-        return "O"
-
-    if 135 <= degrees < 225:
-        return "S"
-
-    return "W"
-
-
-def generate_condition_table(condition):
-    res = ""
-
-    res += (
-        f"<tr><td><b>{gettext('Temperature')}</b></td>"
-        f"<td><b>{condition['temperature_2m']}°C / {c_to_f(condition['temperature_2m'])}°F</b></td></tr>"
+    return WeatherAnswer.Item(
+        location=location,
+        temperature=weather.Temperature(unit="°C", value=data["temperature_2m"]),
+        condition=WMO_TO_CONDITION[data["weather_code"]],
+        feels_like=weather.Temperature(unit="°C", value=data["apparent_temperature"]),
+        wind_from=weather.Compass(data["wind_direction_10m"]),
+        wind_speed=weather.WindSpeed(data["wind_speed_10m"], unit="km/h"),
+        pressure=weather.Pressure(data["pressure_msl"], unit="hPa"),
+        humidity=weather.RelativeHumidity(data["relative_humidity_2m"]),
+        cloud_cover=data["cloud_cover"],
     )
 
-    res += (
-        f"<tr><td>{gettext('Feels like')}</td><td>{condition['apparent_temperature']}°C / "
-        f"{c_to_f(condition['apparent_temperature'])}°F</td></tr>"
-    )
-
-    res += (
-        f"<tr><td>{gettext('Wind')}</td><td>{get_direction(condition['wind_direction_10m'])}, "
-        f"{condition['wind_direction_10m']}° — "
-        f"{condition['wind_speed_10m']} km/h</td></tr>"
-    )
-
-    res += f"<tr><td>{gettext('Cloud cover')}</td><td>{condition['cloud_cover']}%</td>"
-
-    res += f"<tr><td>{gettext('Humidity')}</td><td>{condition['relative_humidity_2m']}%</td></tr>"
-
-    res += f"<tr><td>{gettext('Pressure')}</td><td>{condition['pressure_msl']}hPa</td></tr>"
-
-    return res
-
 
 def response(resp):
-    data = resp.json()
+    location = weather.GeoLocation.by_query(resp.search_params["query"])
 
-    table_content = generate_condition_table(data['current'])
+    res = EngineResults()
+    json_data = resp.json()
 
-    infobox = f"<table><tbody>{table_content}</tbody></table>"
+    weather_answer = WeatherAnswer(
+        current=_weather_data(location, json_data["current"]),
+        service="Open-meteo",
+        # url="https://open-meteo.com/en/docs",
+    )
+
+    for index, time in enumerate(json_data["hourly"]["time"]):
+
+        if time < json_data["current"]["time"]:
+            # Cut off the hours that are already in the past
+            continue
 
-    for index, time in enumerate(data['hourly']['time']):
         hourly_data = {}
+        for key in data_of_interest:
+            hourly_data[key] = json_data["hourly"][key][index]
 
-        for key in data_of_interest.split(","):
-            hourly_data[key] = data['hourly'][key][index]
+        forecast_data = _weather_data(location, hourly_data)
+        forecast_data.datetime = weather.DateTime(datetime.fromtimestamp(time))
+        weather_answer.forecasts.append(forecast_data)
 
-        table_content = generate_condition_table(hourly_data)
-
-        infobox += f"<h3>{datetime.fromtimestamp(time).strftime('%Y-%m-%d %H:%M')}</h3>"
-        infobox += f"<table><tbody>{table_content}</tbody></table>"
-
-    return [{'infobox': 'Open Meteo', 'content': infobox}]
+    res.add(weather_answer)
+    return res

searx/plugins/unit_converter.py

@@ -15,7 +15,7 @@ import babel.numbers
 
 from flask_babel import gettext, get_locale
 
-from searx import data
+from searx.wikidata_units import symbol_to_si
 from searx.plugins import Plugin, PluginInfo
 from searx.result_types import EngineResults
 
@@ -86,132 +86,6 @@ RE_MEASURE = r'''
 '''
 
 
-ADDITIONAL_UNITS = [
-    {
-        "si_name": "Q11579",
-        "symbol": "°C",
-        "to_si": lambda val: val + 273.15,
-        "from_si": lambda val: val - 273.15,
-    },
-    {
-        "si_name": "Q11579",
-        "symbol": "°F",
-        "to_si": lambda val: (val + 459.67) * 5 / 9,
-        "from_si": lambda val: (val * 9 / 5) - 459.67,
-    },
-]
-"""Additional items to convert from a measure unit to a SI unit (vice versa).
-
-.. code:: python
-
-    {
-        "si_name": "Q11579",                 # Wikidata item ID of the SI unit (Kelvin)
-        "symbol": "°C",                      # symbol of the measure unit
-        "to_si": lambda val: val + 273.15,   # convert measure value (val) to SI unit
-        "from_si": lambda val: val - 273.15, # convert SI value (val) measure unit
-    },
-    {
-        "si_name": "Q11573",
-        "symbol": "mi",
-        "to_si": 1609.344,                   # convert measure value (val) to SI unit
-        "from_si": 1 / 1609.344              # convert SI value (val) measure unit
-    },
-
-The values of ``to_si`` and ``from_si`` can be of :py:obj:`float` (a multiplier)
-or a callable_ (val in / converted value returned).
-
-.. _callable: https://docs.python.org/3/glossary.html#term-callable
-"""
-
-
-ALIAS_SYMBOLS = {
-    '°C': ('C',),
-    '°F': ('F',),
-    'mi': ('L',),
-}
-"""Alias symbols for known unit of measure symbols / by example::
-
-    '°C': ('C', ...),  # list of alias symbols for °C (Q69362731)
-    '°F': ('F', ...),  # list of alias symbols for °F (Q99490479)
-    'mi': ('L',),      # list of alias symbols for mi (Q253276)
-"""
-
-
-SYMBOL_TO_SI = []
-
-
-def symbol_to_si():
-    """Generates a list of tuples, each tuple is a measure unit and the fields
-    in the tuple are:
-
-    0. Symbol of the measure unit (e.g. 'mi' for measure unit 'miles' Q253276)
-
-    1. SI name of the measure unit (e.g. Q11573 for SI unit 'metre')
-
-    2. Factor to get SI value from measure unit (e.g. 1mi is equal to SI 1m
-       multiplied by 1609.344)
-
-    3. Factor to get measure value from from SI value (e.g. SI 100m is equal to
-       100mi divided by 1609.344)
-
-    The returned list is sorted, the first items are created from
-    ``WIKIDATA_UNITS``, the second group of items is build from
-    :py:obj:`ADDITIONAL_UNITS` and items created from :py:obj:`ALIAS_SYMBOLS`.
-
-    If you search this list for a symbol, then a match with a symbol from
-    Wikidata has the highest weighting (first hit in the list), followed by the
-    symbols from the :py:obj:`ADDITIONAL_UNITS` and the lowest weighting is
-    given to the symbols resulting from the aliases :py:obj:`ALIAS_SYMBOLS`.
-
-    """
-
-    global SYMBOL_TO_SI  # pylint: disable=global-statement
-    if SYMBOL_TO_SI:
-        return SYMBOL_TO_SI
-
-    # filter out units which can't be normalized to a SI unit and filter out
-    # units without a symbol / arcsecond does not have a symbol
-    # https://www.wikidata.org/wiki/Q829073
-
-    for item in data.WIKIDATA_UNITS.values():
-        if item['to_si_factor'] and item['symbol']:
-            SYMBOL_TO_SI.append(
-                (
-                    item['symbol'],
-                    item['si_name'],
-                    1 / item['to_si_factor'],  # from_si
-                    item['to_si_factor'],  # to_si
-                    item['symbol'],
-                )
-            )
-
-    for item in ADDITIONAL_UNITS:
-        SYMBOL_TO_SI.append(
-            (
-                item['symbol'],
-                item['si_name'],
-                item['from_si'],
-                item['to_si'],
-                item['symbol'],
-            )
-        )
-
-    alias_items = []
-    for item in SYMBOL_TO_SI:
-        for alias in ALIAS_SYMBOLS.get(item[0], ()):
-            alias_items.append(
-                (
-                    alias,
-                    item[1],
-                    item[2],  # from_si
-                    item[3],  # to_si
-                    item[0],  # origin unit
-                )
-            )
-    SYMBOL_TO_SI = SYMBOL_TO_SI + alias_items
-    return SYMBOL_TO_SI
-
-
 def _parse_text_and_convert(from_query, to_query) -> str | None:
 
     # pylint: disable=too-many-branches, too-many-locals

searx/result_types/__init__.py

@@ -13,14 +13,14 @@
 
 from __future__ import annotations
 
-__all__ = ["Result", "MainResult", "KeyValue", "EngineResults", "AnswerSet", "Answer", "Translations"]
+__all__ = ["Result", "MainResult", "KeyValue", "EngineResults", "AnswerSet", "Answer", "Translations", "WeatherAnswer"]
 
 import abc
 
 from searx import enginelib
 
 from ._base import Result, MainResult, LegacyResult
-from .answer import AnswerSet, Answer, Translations
+from .answer import AnswerSet, Answer, Translations, WeatherAnswer
 from .keyvalue import KeyValue
 
 
@@ -35,6 +35,7 @@ class ResultList(list, abc.ABC):
         MainResult = MainResult
         Result = Result
         Translations = Translations
+        WeatherAnswer = WeatherAnswer
 
         # for backward compatibility
         LegacyResult = LegacyResult

searx/result_types/answer.py

@@ -18,6 +18,10 @@ template.
    :members:
    :show-inheritance:
 
+.. autoclass:: WeatherAnswer
+   :members:
+   :show-inheritance:
+
 .. autoclass:: AnswerSet
    :members:
    :show-inheritance:
@@ -26,10 +30,12 @@ template.
 
 from __future__ import annotations
 
-__all__ = ["AnswerSet", "Answer", "Translations"]
+__all__ = ["AnswerSet", "Answer", "Translations", "WeatherAnswer"]
 
+from flask_babel import gettext
 import msgspec
 
+from searx import weather
 from ._base import Result
 
 
@@ -143,3 +149,90 @@ class Translations(BaseAnswer, kw_only=True):
 
         synonyms: list[str] = []
         """List of synonyms for the requested translation."""
+
+
+class WeatherAnswer(BaseAnswer, kw_only=True):
+    """Answer type for weather data."""
+
+    template: str = "answer/weather.html"
+    """The template is located at :origin:`answer/weather.html
+    <searx/templates/simple/answer/weather.html>`"""
+
+    current: WeatherAnswer.Item
+    """Current weather at ``location``."""
+
+    forecasts: list[WeatherAnswer.Item] = []
+    """Weather forecasts for ``location``."""
+
+    service: str = ""
+    """Weather service from which this information was provided."""
+
+    class Item(msgspec.Struct, kw_only=True):
+        """Weather parameters valid for a specific point in time."""
+
+        location: weather.GeoLocation
+        """The geo-location the weather data is from (e.g. `Berlin, Germany`)."""
+
+        temperature: weather.Temperature
+        """Air temperature at 2m above the ground."""
+
+        condition: weather.WeatherConditionType
+        """Standardized designations that summarize the weather situation
+        (e.g. ``light sleet showers and thunder``)."""
+
+        # optional fields
+
+        datetime: weather.DateTime | None = None
+        """Time of the forecast - not needed for the current weather."""
+
+        summary: str | None = None
+        """One-liner about the weather forecast / current weather conditions.
+        If unset, a summary is build up from temperature and current weather
+        conditions.
+        """
+
+        feels_like: weather.Temperature | None = None
+        """Apparent temperature, the temperature equivalent perceived by
+        humans, caused by the combined effects of air temperature, relative
+        humidity and wind speed.  The measure is most commonly applied to the
+        perceived outdoor temperature.
+        """
+
+        pressure: weather.Pressure | None = None
+        """Air pressure at sea level (e.g. 1030 hPa) """
+
+        humidity: weather.RelativeHumidity | None = None
+        """Amount of relative humidity in the air at 2m above the ground. The
+        unit is ``%``, e.g. 60%)
+        """
+
+        wind_from: weather.Compass
+        """The directon which moves towards / direction the wind is coming from."""
+
+        wind_speed: weather.WindSpeed | None = None
+        """Speed of wind / wind speed at 10m above the ground (10 min average)."""
+
+        cloud_cover: int | None = None
+        """Amount of sky covered by clouds / total cloud cover for all heights
+        (cloudiness, unit: %)"""
+
+        # attributes: dict[str, str | int] = {}
+        # """Key-Value dict of additional typeless weather attributes."""
+
+        def __post_init__(self):
+            if not self.summary:
+                self.summary = gettext("{location}: {temperature}, {condition}").format(
+                    location=self.location,
+                    temperature=self.temperature,
+                    condition=gettext(self.condition.capitalize()),
+                )
+
+        @property
+        def url(self) -> str | None:
+            """Determines a `data URL`_ with a symbol for the weather
+            conditions.  If no symbol can be assigned, ``None`` is returned.
+
+            .. _data URL:
+               https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/data
+            """
+            return weather.symbol_url(self.condition)

searx/searxng.msg

@@ -3,8 +3,12 @@
 """A SearXNG message file, see :py:obj:`searx.babel`
 """
 
+import typing
+
 from searx import webutils
 from searx import engines
+from searx.weather import WeatherConditionType
+
 
 __all__ = [
     'CONSTANT_NAMES',
@@ -13,6 +17,7 @@ __all__ = [
     'STYLE_NAMES',
     'BRAND_CUSTOM_LINKS',
     'WEATHER_TERMS',
+    'WEATHER_CONDITIONS',
     'SOCIAL_MEDIA_TERMS',
 ]
 
@@ -85,6 +90,13 @@ WEATHER_TERMS = {
     'WIND': 'Wind',
 }
 
+
+WEATHER_CONDITIONS = [
+    # The capitalized string goes into to i18n/l10n (en: "Clear sky" -> de: "wolkenloser Himmel")
+    msg.capitalize()
+    for msg in typing.get_args(WeatherConditionType)
+]
+
 SOCIAL_MEDIA_TERMS = {
     'SUBSCRIBERS': 'subscribers',
     'POSTS': 'posts',

searx/templates/simple/answer/weather.html

@@ -0,0 +1,62 @@
+{% macro show_weather_data(answer, data) %}
+  <table>
+    <colgroup>
+      <col span="1" class="thumbnail">
+      <col span="1" class="title">
+      <col span="1" class="measured">
+      <col span="1" class="title">
+      <col span="1" class="measured">
+    </colgroup>
+    <tbody>
+      <tr>
+        <td rowspan="4">
+          {%- if data.url %}<img class="symbol" src="{{ data.url }}" title="{{ data.summary }}">{% endif -%}
+        </td>
+      </tr>
+      <tr>
+        <td>{{ _("Temperature") }}:</td>
+        <td>{{ data.temperature.l10n(locale=data.location) }}</td>
+        <td>{{ _("Feels Like") }}:</td>
+        <td>{{ data.feels_like.l10n(locale=data.location) }}</td>
+      </tr>
+      <tr>
+        <td>{{ _("Wind") }}:</td>
+        <td>{{ data.wind_from.l10n(locale=data.location) }}: {{ data.wind_speed.l10n(locale=data.location) }}</td>
+        <td>{{ _("Pressure") }}:</td>
+        <td>{{ data.pressure.l10n(locale=data.location) }}</td>
+      </tr>
+      <tr>
+        <td>{{_("Humidity")}}:</td>
+        <td>{{ data.humidity.l10n(locale=data.location) }}</td>
+        <td></td>
+        <td></td>
+      </tr>
+    </tbody>
+  </table>
+{% endmacro %}
+
+<details class="weather">
+  <summary>
+    <div class="summary"> {{ answer.current.summary }}</div>
+    {{ show_weather_data(answer, answer.current) }}
+  </summary>
+  <div class="weather-forecast">
+    {%- if answer.forecasts -%}
+      <div class="answer-weather-forecasts">
+        {%- for forecast in answer.forecasts -%}
+            <div class="summary">{{ forecast.datetime.l10n(locale=answer.current.location,fmt="short") }} {{ forecast.summary }}</div>
+            {{ show_weather_data(answer, forecast) }}
+        {%- endfor -%}
+      </div>
+    {%- endif -%}
+  </div>
+</details>
+
+{%- if answer.url -%}
+    <a href="{{ answer.url }}" class="answer-url"
+        {%- if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{%- else -%}rel="noreferrer"{%- endif -%}>
+        {{ answer.service }}
+    </a>
+{%- else -%}
+    <span class="answer-url">{{ answer.service }}</span>
+{% endif -%}

searx/templates/simple/preferences/engines.html

@@ -6,7 +6,7 @@
   {%- set ns.checked = false -%}
   {%- if categ == DEFAULT_CATEGORY -%}
     <p>
-      {{- _('This tab does not exists in the user interface, but you can search in these engines by its !bangs.') -}}
+      {{- _('This tab does not exist in the user interface, but you can search with these engines via !bangs.') -}}
       {{- ' ' -}}<a href="{{ url_for('info', pagename='search-syntax') }}">&#9432;</a>
     </p>
   {%- endif -%}

searx/weather.py

@@ -0,0 +1,605 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Implementations used for weather conditions and forecast."""
+# pylint: disable=too-few-public-methods
+from __future__ import annotations
+
+__all__ = [
+    "symbol_url",
+    "Temperature",
+    "Pressure",
+    "WindSpeed",
+    "RelativeHumidity",
+    "Compass",
+    "WeatherConditionType",
+    "DateTime",
+    "GeoLocation",
+]
+
+import typing
+
+import base64
+import datetime
+import dataclasses
+
+from urllib.parse import quote_plus
+
+import babel
+import babel.numbers
+import babel.dates
+import babel.languages
+
+from searx import network
+from searx.cache import ExpireCache, ExpireCacheCfg
+from searx.extended_types import sxng_request
+from searx.wikidata_units import convert_to_si, convert_from_si
+
+WEATHER_DATA_CACHE: ExpireCache = None  # type: ignore
+"""A simple cache for weather data (geo-locations, icons, ..)"""
+
+YR_WEATHER_SYMBOL_URL = "https://raw.githubusercontent.com/nrkno/yr-weather-symbols/refs/heads/master/symbols/outline"
+
+
+def get_WEATHER_DATA_CACHE():
+
+    global WEATHER_DATA_CACHE  # pylint: disable=global-statement
+
+    if WEATHER_DATA_CACHE is None:
+        WEATHER_DATA_CACHE = ExpireCache.build_cache(
+            ExpireCacheCfg(
+                name="WEATHER_DATA_CACHE",
+                MAX_VALUE_LEN=1024 * 200,  # max. 200kB per icon (icons have most often 10-20kB)
+                MAXHOLD_TIME=60 * 60 * 24 * 7 * 4,  # 4 weeks
+            )
+        )
+    return WEATHER_DATA_CACHE
+
+
+def _get_sxng_locale_tag() -> str:
+    # The function should return a locale (the sxng-tag: de-DE.en-US, ..) that
+    # can later be used to format and convert measured values for the output of
+    # weather data to the user.
+    #
+    # In principle, SearXNG only has two possible parameters for determining
+    # the locale: the UI language or the search- language/region.  Since the
+    # conversion of weather data and time information is usually
+    # region-specific, the UI language is not suitable.
+    #
+    # It would probably be ideal to use the user's geolocation, but this will
+    # probably never be available in SearXNG (privacy critical).
+    #
+    # Therefore, as long as no "better" parameters are available, this function
+    # returns a locale based on the search region.
+
+    # pylint: disable=import-outside-toplevel,disable=cyclic-import
+    from searx import query
+    from searx.preferences import ClientPref
+
+    query = query.RawTextQuery(sxng_request.form.get("q", ""), [])
+    if query.languages and query.languages[0] not in ["all", "auto"]:
+        return query.languages[0]
+
+    search_lang = sxng_request.form.get("language")
+    if search_lang and search_lang not in ["all", "auto"]:
+        return search_lang
+
+    client_pref = ClientPref.from_http_request(sxng_request)
+    search_lang = client_pref.locale_tag
+    if search_lang and search_lang not in ["all", "auto"]:
+        return search_lang
+    return "en"
+
+
+def symbol_url(condition: WeatherConditionType) -> str | None:
+    """Returns ``data:`` URL for the weather condition symbol or ``None`` if
+    the condition is not of type :py:obj:`WeatherConditionType`.
+
+    If symbol (SVG) is not already in the :py:obj:`WEATHER_DATA_CACHE` its
+    fetched from https://github.com/nrkno/yr-weather-symbols
+    """
+    # Symbols for darkmode/lightmode? .. and day/night symbols? .. for the
+    # latter we need a geopoint (critical in sense of privacy)
+
+    fname = YR_WEATHER_SYMBOL_MAP.get(condition)
+    if fname is None:
+        return None
+
+    ctx = "weather_symbol_url"
+    cache = get_WEATHER_DATA_CACHE()
+    origin_url = f"{YR_WEATHER_SYMBOL_URL}/{fname}.svg"
+
+    data_url = cache.get(origin_url, ctx=ctx)
+    if data_url is not None:
+        return data_url
+
+    response = network.get(origin_url, timeout=3)
+    if response.status_code == 200:
+        mimetype = response.headers['Content-Type']
+        data_url = f"data:{mimetype};base64,{str(base64.b64encode(response.content), 'utf-8')}"
+        cache.set(key=origin_url, value=data_url, expire=None, ctx=ctx)
+    return data_url
+
+
+@dataclasses.dataclass
+class GeoLocation:
+    """Minimal implementation of Geocoding."""
+
+    # The type definition was based on the properties from the geocoding API of
+    # open-meteo.
+    #
+    # - https://open-meteo.com/en/docs/geocoding-api
+    # - https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+    # - https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2
+
+    name: str
+    latitude: float  # Geographical WGS84 coordinates of this location
+    longitude: float
+    elevation: float  # Elevation above mean sea level of this location
+    country_code: str  # 2-Character ISO-3166-1 alpha2 country code. E.g. DE for Germany
+    timezone: str  # Time zone using time zone database definitions
+
+    def __str__(self):
+        return self.name
+
+    def locale(self) -> babel.Locale:
+
+        # by region of the search language
+        sxng_tag = _get_sxng_locale_tag()
+        if "-" in sxng_tag:
+            locale = babel.Locale.parse(_get_sxng_locale_tag(), sep='-')
+            return locale
+
+        # by most popular language in the region (country code)
+        for lang in babel.languages.get_official_languages(self.country_code):
+            try:
+                locale = babel.Locale.parse(f"{lang}_{self.country_code}")
+                return locale
+            except babel.UnknownLocaleError:
+                continue
+
+        # No locale could be determined.  This does not actually occur, but if
+        # it does, the English language is used by default.  But not region US.
+        # US has some units that are only used in US but not in the rest of the
+        # world (e.g. °F instead of °C)
+        return babel.Locale("en", territory="DE")
+
+    @classmethod
+    def by_query(cls, search_term: str) -> GeoLocation:
+        """Factory method to get a GeoLocation object by a search term.  If no
+        location can be determined for the search term, a :py:obj:`ValueError`
+        is thrown.
+        """
+
+        ctx = "weather_geolocation_by_query"
+        cache = get_WEATHER_DATA_CACHE()
+        geo_props = cache.get(search_term, ctx=ctx)
+
+        if not geo_props:
+            geo_props = cls._query_open_meteo(search_term=search_term)
+            cache.set(key=search_term, value=geo_props, expire=None, ctx=ctx)
+
+        return cls(**geo_props)
+
+    @classmethod
+    def _query_open_meteo(cls, search_term: str) -> dict:
+        url = f"https://geocoding-api.open-meteo.com/v1/search?name={quote_plus(search_term)}"
+        resp = network.get(url, timeout=3)
+        if resp.status_code != 200:
+            raise ValueError(f"unknown geo location: '{search_term}'")
+        results = resp.json().get("results")
+        if not results:
+            raise ValueError(f"unknown geo location: '{search_term}'")
+        location = results[0]
+        return {field.name: location[field.name] for field in dataclasses.fields(cls)}
+
+
+DateTimeFormats = typing.Literal["full", "long", "medium", "short"]
+
+
+class DateTime:
+    """Class to represent date & time.  Essentially, it is a wrapper that
+    conveniently combines :py:obj:`datetime.datetime` and
+    :py:obj:`babel.dates.format_datetime`.  A conversion of time zones is not
+    provided (in the current version).
+    """
+
+    def __init__(self, time: datetime.datetime):
+        self.datetime = time
+
+    def __str__(self):
+        return self.l10n()
+
+    def l10n(
+        self,
+        fmt: DateTimeFormats | str = "medium",
+        locale: babel.Locale | GeoLocation | None = None,
+    ) -> str:
+        """Localized representation of date & time."""
+        if isinstance(locale, GeoLocation):
+            locale = locale.locale()
+        elif locale is None:
+            locale = babel.Locale.parse(_get_sxng_locale_tag(), sep='-')
+        return babel.dates.format_datetime(self.datetime, format=fmt, locale=locale)
+
+
+class Temperature:
+    """Class for converting temperature units and for string representation of
+    measured values."""
+
+    si_name = "Q11579"
+
+    Units = typing.Literal["°C", "°F", "K"]
+    """Supported temperature units."""
+
+    units = list(typing.get_args(Units))
+
+    def __init__(self, value: float, unit: Units):
+        if unit not in self.units:
+            raise ValueError(f"invalid unit: {unit}")
+        self.si: float = convert_to_si(  # pylint: disable=invalid-name
+            si_name=self.si_name,
+            symbol=unit,
+            value=value,
+        )
+
+    def __str__(self):
+        return self.l10n()
+
+    def value(self, unit: Units) -> float:
+        return convert_from_si(si_name=self.si_name, symbol=unit, value=self.si)
+
+    def l10n(
+        self,
+        unit: Units | None = None,
+        locale: babel.Locale | GeoLocation | None = None,
+        template: str = "{value} {unit}",
+        num_pattern: str = "#,##0",
+    ) -> str:
+        """Localized representation of a measured value.
+
+        If the ``unit`` is not set, an attempt is made to determine a ``unit``
+        matching the territory of the ``locale``.  If the locale is not set, an
+        attempt is made to determine it from the HTTP request.
+
+        The value is converted into the respective unit before formatting.
+
+        The argument ``num_pattern`` is used to determine the string formatting
+        of the numerical value:
+
+        - https://babel.pocoo.org/en/latest/numbers.html#pattern-syntax
+        - https://unicode.org/reports/tr35/tr35-numbers.html#Number_Format_Patterns
+
+        The argument ``template`` specifies how the **string formatted** value
+        and unit are to be arranged.
+
+        - `Format Specification Mini-Language
+          <https://docs.python.org/3/library/string.html#format-specification-mini-language>`.
+        """
+
+        if isinstance(locale, GeoLocation):
+            locale = locale.locale()
+        elif locale is None:
+            locale = babel.Locale.parse(_get_sxng_locale_tag(), sep='-')
+
+        if unit is None:  # unit by territory
+            unit = "°C"
+            if locale.territory in ["US"]:
+                unit = "°F"
+        val_str = babel.numbers.format_decimal(self.value(unit), locale=locale, format=num_pattern)
+        return template.format(value=val_str, unit=unit)
+
+
+class Pressure:
+    """Class for converting pressure units and for string representation of
+    measured values."""
+
+    si_name = "Q44395"
+
+    Units = typing.Literal["Pa", "hPa", "cm Hg", "bar"]
+    """Supported units."""
+
+    units = list(typing.get_args(Units))
+
+    def __init__(self, value: float, unit: Units):
+        if unit not in self.units:
+            raise ValueError(f"invalid unit: {unit}")
+        # pylint: disable=invalid-name
+        self.si: float = convert_to_si(si_name=self.si_name, symbol=unit, value=value)
+
+    def __str__(self):
+        return self.l10n()
+
+    def value(self, unit: Units) -> float:
+        return convert_from_si(si_name=self.si_name, symbol=unit, value=self.si)
+
+    def l10n(
+        self,
+        unit: Units | None = None,
+        locale: babel.Locale | GeoLocation | None = None,
+        template: str = "{value} {unit}",
+        num_pattern: str = "#,##0",
+    ) -> str:
+        if isinstance(locale, GeoLocation):
+            locale = locale.locale()
+        elif locale is None:
+            locale = babel.Locale.parse(_get_sxng_locale_tag(), sep='-')
+
+        if unit is None:  # unit by territory?
+            unit = "hPa"
+
+        val_str = babel.numbers.format_decimal(self.value(unit), locale=locale, format=num_pattern)
+        return template.format(value=val_str, unit=unit)
+
+
+class WindSpeed:
+    """Class for converting speed or velocity units and for string
+    representation of measured values.
+
+    .. hint::
+
+       Working with unit ``Bft`` (:py:obj:`searx.wikidata_units.Beaufort`) will
+       throw a :py:obj:`ValueError` for egative values or values greater 16 Bft
+       (55.6 m/s)
+    """
+
+    si_name = "Q182429"
+
+    Units = typing.Literal["m/s", "km/h", "kn", "mph", "mi/h", "Bft"]
+    """Supported units."""
+
+    units = list(typing.get_args(Units))
+
+    def __init__(self, value: float, unit: Units):
+        if unit not in self.units:
+            raise ValueError(f"invalid unit: {unit}")
+        # pylint: disable=invalid-name
+        self.si: float = convert_to_si(si_name=self.si_name, symbol=unit, value=value)
+
+    def __str__(self):
+        return self.l10n()
+
+    def value(self, unit: Units) -> float:
+        return convert_from_si(si_name=self.si_name, symbol=unit, value=self.si)
+
+    def l10n(
+        self,
+        unit: Units | None = None,
+        locale: babel.Locale | GeoLocation | None = None,
+        template: str = "{value} {unit}",
+        num_pattern: str = "#,##0",
+    ) -> str:
+        if isinstance(locale, GeoLocation):
+            locale = locale.locale()
+        elif locale is None:
+            locale = babel.Locale.parse(_get_sxng_locale_tag(), sep='-')
+
+        if unit is None:  # unit by territory?
+            unit = "m/s"
+
+        val_str = babel.numbers.format_decimal(self.value(unit), locale=locale, format=num_pattern)
+        return template.format(value=val_str, unit=unit)
+
+
+class RelativeHumidity:
+    """Amount of relative humidity in the air. The unit is ``%``"""
+
+    Units = typing.Literal["%"]
+    """Supported unit."""
+
+    units = list(typing.get_args(Units))
+
+    def __init__(self, humidity: float):
+        self.humidity = humidity
+
+    def __str__(self):
+        return self.l10n()
+
+    def value(self) -> float:
+        return self.humidity
+
+    def l10n(
+        self,
+        locale: babel.Locale | GeoLocation | None = None,
+        template: str = "{value}{unit}",
+        num_pattern: str = "#,##0",
+    ) -> str:
+        if isinstance(locale, GeoLocation):
+            locale = locale.locale()
+        elif locale is None:
+            locale = babel.Locale.parse(_get_sxng_locale_tag(), sep='-')
+
+        unit = "%"
+        val_str = babel.numbers.format_decimal(self.value(), locale=locale, format=num_pattern)
+        return template.format(value=val_str, unit=unit)
+
+
+class Compass:
+    """Class for converting compass points and azimuth values (360°)"""
+
+    Units = typing.Literal["°", "Point"]
+
+    Point = typing.Literal[
+        "N", "NNE", "NE", "ENE", "E", "ESE", "SE", "SSE", "S", "SSW", "SW", "WSW", "W", "WNW", "NW", "NNW"
+    ]
+    """Compass point type definition"""
+
+    TURN = 360.0
+    """Full turn (360°)"""
+
+    POINTS = list(typing.get_args(Point))
+    """Compass points."""
+
+    RANGE = TURN / len(POINTS)
+    """Angle sector of a compass point"""
+
+    def __init__(self, azimuth: float | int | Point):
+        if isinstance(azimuth, str):
+            if azimuth not in self.POINTS:
+                raise ValueError(f"Invalid compass point: {azimuth}")
+            azimuth = self.POINTS.index(azimuth) * self.RANGE
+        self.azimuth = azimuth % self.TURN
+
+    def __str__(self):
+        return self.l10n()
+
+    def value(self, unit: Units):
+        if unit == "Point":
+            return self.point(self.azimuth)
+        if unit == "°":
+            return self.azimuth
+        raise ValueError(f"unknown unit: {unit}")
+
+    @classmethod
+    def point(cls, azimuth: float | int) -> Point:
+        """Returns the compass point to an azimuth value."""
+        azimuth = azimuth % cls.TURN
+        # The angle sector of a compass point starts 1/2 sector range before
+        # and after compass point (example: "N" goes from -11.25° to +11.25°)
+        azimuth = azimuth - cls.RANGE / 2
+        idx = int(azimuth // cls.RANGE)
+        return cls.POINTS[idx]
+
+    def l10n(
+        self,
+        unit: Units = "Point",
+        locale: babel.Locale | GeoLocation | None = None,
+        template: str = "{value}{unit}",
+        num_pattern: str = "#,##0",
+    ) -> str:
+        if isinstance(locale, GeoLocation):
+            locale = locale.locale()
+        elif locale is None:
+            locale = babel.Locale.parse(_get_sxng_locale_tag(), sep='-')
+
+        if unit == "Point":
+            val_str = self.value(unit)
+            return template.format(value=val_str, unit="")
+
+        val_str = babel.numbers.format_decimal(self.value(unit), locale=locale, format=num_pattern)
+        return template.format(value=val_str, unit=unit)
+
+
+WeatherConditionType = typing.Literal[
+    # The capitalized string goes into to i18n/l10n (en: "Clear sky" -> de: "wolkenloser Himmel")
+    "clear sky",
+    "cloudy",
+    "fair",
+    "fog",
+    "heavy rain and thunder",
+    "heavy rain showers and thunder",
+    "heavy rain showers",
+    "heavy rain",
+    "heavy sleet and thunder",
+    "heavy sleet showers and thunder",
+    "heavy sleet showers",
+    "heavy sleet",
+    "heavy snow and thunder",
+    "heavy snow showers and thunder",
+    "heavy snow showers",
+    "heavy snow",
+    "light rain and thunder",
+    "light rain showers and thunder",
+    "light rain showers",
+    "light rain",
+    "light sleet and thunder",
+    "light sleet showers and thunder",
+    "light sleet showers",
+    "light sleet",
+    "light snow and thunder",
+    "light snow showers and thunder",
+    "light snow showers",
+    "light snow",
+    "partly cloudy",
+    "rain and thunder",
+    "rain showers and thunder",
+    "rain showers",
+    "rain",
+    "sleet and thunder",
+    "sleet showers and thunder",
+    "sleet showers",
+    "sleet",
+    "snow and thunder",
+    "snow showers and thunder",
+    "snow showers",
+    "snow",
+]
+"""Standardized designations for weather conditions.  The designators were
+taken from a collaboration between NRK and Norwegian Meteorological Institute
+(yr.no_).  `Weather symbols`_ can be assigned to the identifiers
+(weathericons_) and they are included in the translation (i18n/l10n
+:origin:`searx/searxng.msg`).
+
+.. _yr.no: https://www.yr.no/en
+.. _Weather symbols: https://github.com/nrkno/yr-weather-symbols
+.. _weathericons: https://github.com/metno/weathericons
+"""
+
+YR_WEATHER_SYMBOL_MAP = {
+    "clear sky": "01d",  # 01d clearsky_day
+    "fair": "02d",  # 02d fair_day
+    "partly cloudy": "03d",  # 03d partlycloudy_day
+    "cloudy": "04",  # 04 cloudy
+    "light rain showers": "40d",  # 40d lightrainshowers_day
+    "rain showers": "05d",  # 05d rainshowers_day
+    "heavy rain showers": "41d",  # 41d heavyrainshowers_day
+    "light rain showers and thunder": "24d",  # 24d lightrainshowersandthunder_day
+    "rain showers and thunder": "06d",  # 06d rainshowersandthunder_day
+    "heavy rain showers and thunder": "25d",  # 25d heavyrainshowersandthunder_day
+    "light sleet showers": "42d",  # 42d lightsleetshowers_day
+    "sleet showers": "07d",  # 07d sleetshowers_day
+    "heavy sleet showers": "43d",  # 43d heavysleetshowers_day
+    "light sleet showers and thunder": "26d",  # 26d lightssleetshowersandthunder_day
+    "sleet showers and thunder": "20d",  # 20d sleetshowersandthunder_day
+    "heavy sleet showers and thunder": "27d",  # 27d heavysleetshowersandthunder_day
+    "light snow showers": "44d",  # 44d lightsnowshowers_day
+    "snow showers": "08d",  # 08d snowshowers_day
+    "heavy snow showers": "45d",  # 45d heavysnowshowers_day
+    "light snow showers and thunder": "28d",  # 28d lightssnowshowersandthunder_day
+    "snow showers and thunder": "21d",  # 21d snowshowersandthunder_day
+    "heavy snow showers and thunder": "29d",  # 29d heavysnowshowersandthunder_day
+    "light rain": "46",  # 46 lightrain
+    "rain": "09",  # 09 rain
+    "heavy rain": "10",  # 10 heavyrain
+    "light rain and thunder": "30",  # 30 lightrainandthunder
+    "rain and thunder": "22",  # 22 rainandthunder
+    "heavy rain and thunder": "11",  # 11 heavyrainandthunder
+    "light sleet": "47",  # 47 lightsleet
+    "sleet": "12",  # 12 sleet
+    "heavy sleet": "48",  # 48 heavysleet
+    "light sleet and thunder": "31",  # 31 lightsleetandthunder
+    "sleet and thunder": "23",  # 23 sleetandthunder
+    "heavy sleet and thunder": "32",  # 32 heavysleetandthunder
+    "light snow": "49",  # 49 lightsnow
+    "snow": "13",  # 13 snow
+    "heavy snow": "50",  # 50 heavysnow
+    "light snow and thunder": "33",  # 33 lightsnowandthunder
+    "snow and thunder": "14",  # 14 snowandthunder
+    "heavy snow and thunder": "34",  # 34 heavysnowandthunder
+    "fog": "15",  # 15 fog
+}
+"""Map a :py:obj:`WeatherConditionType` to a `YR weather symbol`_
+
+.. code::
+
+   base_url = "https://raw.githubusercontent.com/nrkno/yr-weather-symbols/refs/heads/master/symbols"
+   icon_url = f"{base_url}/outline/{YR_WEATHER_SYMBOL_MAP['sleet showers']}.svg"
+
+.. _YR weather symbol: https://github.com/nrkno/yr-weather-symbols/blob/master/locales/en.json
+
+"""
+
+if __name__ == "__main__":
+
+    # test: fetch all symbols of the type catalog ..
+    for c in typing.get_args(WeatherConditionType):
+        symbol_url(condition=c)
+
+    _cache = get_WEATHER_DATA_CACHE()
+    title = "cached weather condition symbols"
+    print(title)
+    print("=" * len(title))
+    print(_cache.state().report())
+    print()
+    title = f"properties of {_cache.cfg.name}"
+    print(title)
+    print("=" * len(title))
+    print(str(_cache.properties))  # type: ignore

searx/wikidata_units.py

@@ -0,0 +1,292 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Unit conversion on the basis of `SPARQL/WIKIDATA Precision, Units and
+Coordinates`_
+
+.. _SPARQL/WIKIDATA Precision, Units and Coordinates:
+   https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates#Quantities
+"""
+from __future__ import annotations
+
+__all__ = ["convert_from_si", "convert_to_si", "symbol_to_si"]
+
+import collections
+
+from searx import data
+from searx.engines import wikidata
+
+
+class Beaufort:
+    """The mapping of the Beaufort_ contains values from 0 to 16 (55.6 m/s),
+    wind speeds greater than 200km/h (55.6 m/s) are given as 17 Bft. Thats why
+    a value of 17 Bft cannot be converted to SI.
+
+    .. hint::
+
+       Negative values or values greater 16 Bft (55.6 m/s) will throw a
+       :py:obj:`ValueError`.
+
+    _Beaufort: https://en.wikipedia.org/wiki/Beaufort_scale
+    """
+
+    # fmt: off
+    scale: list[float] = [
+         0.2,  1.5,  3.3,  5.4,  7.9,
+        10.7, 13.8, 17.1, 20.7, 24.4,
+        28.4, 32.6, 32.7, 41.1, 45.8,
+        50.8, 55.6
+    ]
+    # fmt: on
+
+    @classmethod
+    def from_si(cls, value) -> float:
+        if value < 0 or value > 55.6:
+            raise ValueError(f"invalid value {value} / the Beaufort scales from 0 to 16 (55.6 m/s)")
+        bft = 0
+        for bft, mps in enumerate(cls.scale):
+            if mps >= value:
+                break
+        return bft
+
+    @classmethod
+    def to_si(cls, value) -> float:
+        idx = round(value)
+        if idx < 0 or idx > 16:
+            raise ValueError(f"invalid value {value} / the Beaufort scales from 0 to 16 (55.6 m/s)")
+        return cls.scale[idx]
+
+
+ADDITIONAL_UNITS = [
+    {
+        "si_name": "Q11579",
+        "symbol": "°C",
+        "to_si": lambda val: val + 273.15,
+        "from_si": lambda val: val - 273.15,
+    },
+    {
+        "si_name": "Q11579",
+        "symbol": "°F",
+        "to_si": lambda val: (val + 459.67) * 5 / 9,
+        "from_si": lambda val: (val * 9 / 5) - 459.67,
+    },
+    {
+        "si_name": "Q182429",
+        "symbol": "Bft",
+        "to_si": Beaufort.to_si,
+        "from_si": Beaufort.from_si,
+    },
+]
+"""Additional items to convert from a measure unit to a SI unit (vice versa).
+
+.. code:: python
+
+    {
+        "si_name": "Q11579",                 # Wikidata item ID of the SI unit (Kelvin)
+        "symbol": "°C",                      # symbol of the measure unit
+        "to_si": lambda val: val + 273.15,   # convert measure value (val) to SI unit
+        "from_si": lambda val: val - 273.15, # convert SI value (val) measure unit
+    },
+    {
+        "si_name": "Q11573",
+        "symbol": "mi",
+        "to_si": 1609.344,                   # convert measure value (val) to SI unit
+        "from_si": 1 / 1609.344              # convert SI value (val) measure unit
+    },
+
+The values of ``to_si`` and ``from_si`` can be of :py:obj:`float` (a multiplier)
+or a callable_ (val in / converted value returned).
+
+.. _callable: https://docs.python.org/3/glossary.html#term-callable
+"""
+
+
+ALIAS_SYMBOLS = {
+    '°C': ('C',),
+    '°F': ('F',),
+    'mi': ('L',),
+    'Bft': ('bft',),
+}
+"""Alias symbols for known unit of measure symbols / by example::
+
+    '°C': ('C', ...),  # list of alias symbols for °C (Q69362731)
+    '°F': ('F', ...),  # list of alias symbols for °F (Q99490479)
+    'mi': ('L',),      # list of alias symbols for mi (Q253276)
+"""
+
+
+SYMBOL_TO_SI = []
+UNITS_BY_SI_NAME: dict = {}
+
+
+def convert_from_si(si_name: str, symbol: str, value: float | int) -> float:
+    from_si = units_by_si_name(si_name)[symbol][pos_from_si]
+    if isinstance(from_si, (float, int)):
+        value = float(value) * from_si
+    else:
+        value = from_si(float(value))
+    return value
+
+
+def convert_to_si(si_name: str, symbol: str, value: float | int) -> float:
+    to_si = units_by_si_name(si_name)[symbol][pos_to_si]
+    if isinstance(to_si, (float, int)):
+        value = float(value) * to_si
+    else:
+        value = to_si(float(value))
+    return value
+
+
+def units_by_si_name(si_name):
+
+    global UNITS_BY_SI_NAME  # pylint: disable=global-statement,global-variable-not-assigned
+    if UNITS_BY_SI_NAME:
+        return UNITS_BY_SI_NAME[si_name]
+
+    # build the catalog ..
+    for item in symbol_to_si():
+
+        item_si_name = item[pos_si_name]
+        item_symbol = item[pos_symbol]
+
+        by_symbol = UNITS_BY_SI_NAME.get(item_si_name)
+        if by_symbol is None:
+            by_symbol = {}
+            UNITS_BY_SI_NAME[item_si_name] = by_symbol
+        by_symbol[item_symbol] = item
+
+    return UNITS_BY_SI_NAME[si_name]
+
+
+pos_symbol = 0  # (alias) symbol
+pos_si_name = 1  # si_name
+pos_from_si = 2  # from_si
+pos_to_si = 3  # to_si
+pos_symbol = 4  # standardized symbol
+
+
+def symbol_to_si():
+    """Generates a list of tuples, each tuple is a measure unit and the fields
+    in the tuple are:
+
+    0. Symbol of the measure unit (e.g. 'mi' for measure unit 'miles' Q253276)
+
+    1. SI name of the measure unit (e.g. Q11573 for SI unit 'metre')
+
+    2. Factor to get SI value from measure unit (e.g. 1mi is equal to SI 1m
+       multiplied by 1609.344)
+
+    3. Factor to get measure value from from SI value (e.g. SI 100m is equal to
+       100mi divided by 1609.344)
+
+    The returned list is sorted, the first items are created from
+    ``WIKIDATA_UNITS``, the second group of items is build from
+    :py:obj:`ADDITIONAL_UNITS` and items created from :py:obj:`ALIAS_SYMBOLS`.
+
+    If you search this list for a symbol, then a match with a symbol from
+    Wikidata has the highest weighting (first hit in the list), followed by the
+    symbols from the :py:obj:`ADDITIONAL_UNITS` and the lowest weighting is
+    given to the symbols resulting from the aliases :py:obj:`ALIAS_SYMBOLS`.
+
+    """
+
+    global SYMBOL_TO_SI  # pylint: disable=global-statement
+    if SYMBOL_TO_SI:
+        return SYMBOL_TO_SI
+
+    # filter out units which can't be normalized to a SI unit and filter out
+    # units without a symbol / arcsecond does not have a symbol
+    # https://www.wikidata.org/wiki/Q829073
+
+    for item in data.WIKIDATA_UNITS.values():
+        if item['to_si_factor'] and item['symbol']:
+            SYMBOL_TO_SI.append(
+                (
+                    item['symbol'],
+                    item['si_name'],
+                    1 / item['to_si_factor'],  # from_si
+                    item['to_si_factor'],  # to_si
+                    item['symbol'],
+                )
+            )
+
+    for item in ADDITIONAL_UNITS:
+        SYMBOL_TO_SI.append(
+            (
+                item['symbol'],
+                item['si_name'],
+                item['from_si'],
+                item['to_si'],
+                item['symbol'],
+            )
+        )
+
+    alias_items = []
+    for item in SYMBOL_TO_SI:
+        for alias in ALIAS_SYMBOLS.get(item[0], ()):
+            alias_items.append(
+                (
+                    alias,
+                    item[1],
+                    item[2],  # from_si
+                    item[3],  # to_si
+                    item[0],  # origin unit
+                )
+            )
+    SYMBOL_TO_SI = SYMBOL_TO_SI + alias_items
+    return SYMBOL_TO_SI
+
+
+# the response contains duplicate ?item with the different ?symbol
+# "ORDER BY ?item DESC(?rank) ?symbol" provides a deterministic result
+# even if a ?item has different ?symbol of the same rank.
+# A deterministic result
+# see:
+# * https://www.wikidata.org/wiki/Help:Ranking
+# * https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format ("Statement representation" section)
+# * https://w.wiki/32BT
+# * https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates#Quantities
+#   see the result for https://www.wikidata.org/wiki/Q11582
+#   there are multiple symbols the same rank
+
+SARQL_REQUEST = """
+SELECT DISTINCT ?item ?symbol ?tosi ?tosiUnit
+WHERE
+{
+  ?item wdt:P31/wdt:P279 wd:Q47574 .
+  ?item p:P5061 ?symbolP .
+  ?symbolP ps:P5061 ?symbol ;
+           wikibase:rank ?rank .
+  OPTIONAL {
+    ?item p:P2370 ?tosistmt .
+    ?tosistmt psv:P2370 ?tosinode .
+    ?tosinode wikibase:quantityAmount ?tosi .
+    ?tosinode wikibase:quantityUnit ?tosiUnit .
+  }
+  FILTER(LANG(?symbol) = "en").
+}
+ORDER BY ?item DESC(?rank) ?symbol
+"""
+
+
+def fetch_units():
+    """Fetch units from Wikidata.  Function is used to update persistence of
+    :py:obj:`searx.data.WIKIDATA_UNITS`."""
+
+    results = collections.OrderedDict()
+    response = wikidata.send_wikidata_query(SARQL_REQUEST)
+    for unit in response['results']['bindings']:
+
+        symbol = unit['symbol']['value']
+        name = unit['item']['value'].rsplit('/', 1)[1]
+        si_name = unit.get('tosiUnit', {}).get('value', '')
+        if si_name:
+            si_name = si_name.rsplit('/', 1)[1]
+
+        to_si_factor = unit.get('tosi', {}).get('value', '')
+        if name not in results:
+            # ignore duplicate: always use the first one
+            results[name] = {
+                'symbol': symbol,
+                'si_name': si_name if si_name else None,
+                'to_si_factor': float(to_si_factor) if to_si_factor else None,
+            }
+    return results

searxng_extra/update/update_wikidata_units.py

@@ -8,76 +8,15 @@ Output file: :origin:`searx/data/wikidata_units.json` (:origin:`CI Update data
 """
 
 import json
-import collections
 
-# set path
-from os.path import join
-
-from searx import searx_dir
 from searx.engines import wikidata, set_loggers
 from searx.data import data_dir
+from searx.wikidata_units import fetch_units
 
 DATA_FILE = data_dir / 'wikidata_units.json'
-
 set_loggers(wikidata, 'wikidata')
 
-# the response contains duplicate ?item with the different ?symbol
-# "ORDER BY ?item DESC(?rank) ?symbol" provides a deterministic result
-# even if a ?item has different ?symbol of the same rank.
-# A deterministic result
-# see:
-# * https://www.wikidata.org/wiki/Help:Ranking
-# * https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format ("Statement representation" section)
-# * https://w.wiki/32BT
-# * https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates#Quantities
-#   see the result for https://www.wikidata.org/wiki/Q11582
-#   there are multiple symbols the same rank
-SARQL_REQUEST = """
-SELECT DISTINCT ?item ?symbol ?tosi ?tosiUnit
-WHERE
-{
-  ?item wdt:P31/wdt:P279 wd:Q47574 .
-  ?item p:P5061 ?symbolP .
-  ?symbolP ps:P5061 ?symbol ;
-           wikibase:rank ?rank .
-  OPTIONAL {
-    ?item p:P2370 ?tosistmt .
-    ?tosistmt psv:P2370 ?tosinode .
-    ?tosinode wikibase:quantityAmount ?tosi .
-    ?tosinode wikibase:quantityUnit ?tosiUnit .
-  }
-  FILTER(LANG(?symbol) = "en").
-}
-ORDER BY ?item DESC(?rank) ?symbol
-"""
-
-
-def get_data():
-    results = collections.OrderedDict()
-    response = wikidata.send_wikidata_query(SARQL_REQUEST)
-    for unit in response['results']['bindings']:
-
-        symbol = unit['symbol']['value']
-        name = unit['item']['value'].rsplit('/', 1)[1]
-        si_name = unit.get('tosiUnit', {}).get('value', '')
-        if si_name:
-            si_name = si_name.rsplit('/', 1)[1]
-
-        to_si_factor = unit.get('tosi', {}).get('value', '')
-        if name not in results:
-            # ignore duplicate: always use the first one
-            results[name] = {
-                'symbol': symbol,
-                'si_name': si_name if si_name else None,
-                'to_si_factor': float(to_si_factor) if to_si_factor else None,
-            }
-    return results
-
-
-def get_wikidata_units_filename():
-    return join(join(searx_dir, "data"), "")
-
 
 if __name__ == '__main__':
     with DATA_FILE.open('w', encoding="utf8") as f:
-        json.dump(get_data(), f, indent=4, sort_keys=True, ensure_ascii=False)
+        json.dump(fetch_units(), f, indent=4, sort_keys=True, ensure_ascii=False)