[mod] data: implement a simple currencies (SQL) database (#4836)

To reduce the memory footprint, this patch no longer loads the JSON data
completely into memory.  Instead, there is an SQL database based on
`ExpireCacheSQLite`.

The class CurrenciesDB is a simple DB application that encapsulates the
DB (queries and initialization) and provides convenient methods like
`name_to_iso4217` and `iso4217_to_name`.

Related:

- https://github.com/searxng/searxng/discussions/1892
- https://github.com/searxng/searxng/pull/3458#issuecomment-2900807671
- https://github.com/searxng/searxng/pull/4650

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/data/__init__.py

@@ -4,21 +4,17 @@
   make data.all
 
 """
-
+from __future__ import annotations
 
 __all__ = ["ahmia_blacklist_loader"]
 
 import json
-from pathlib import Path
 import typing
 
-from searx import logger
+from .core import log, data_dir
+from .currencies import CurrenciesDB
 
-log = logger.getChild("data")
-
-data_dir = Path(__file__).parent
-
-CURRENCIES: dict[str, typing.Any]
+CURRENCIES: CurrenciesDB
 USER_AGENTS: dict[str, typing.Any]
 EXTERNAL_URLS: dict[str, typing.Any]
 WIKIDATA_UNITS: dict[str, typing.Any]
@@ -29,7 +25,7 @@ ENGINE_TRAITS: dict[str, typing.Any]
 LOCALES: dict[str, typing.Any]
 
 lazy_globals = {
-    "CURRENCIES": None,
+    "CURRENCIES": CurrenciesDB(),
     "USER_AGENTS": None,
     "EXTERNAL_URLS": None,
     "WIKIDATA_UNITS": None,
@@ -41,7 +37,6 @@ lazy_globals = {
 }
 
 data_json_files = {
-    "CURRENCIES": "currencies.json",
     "USER_AGENTS": "useragents.json",
     "EXTERNAL_URLS": "external_urls.json",
     "WIKIDATA_UNITS": "wikidata_units.json",
@@ -63,6 +58,7 @@ def __getattr__(name):
         return data
 
     log.debug("init searx.data.%s", name)
+
     with open(data_dir / data_json_files[name], encoding='utf-8') as f:
         lazy_globals[name] = json.load(f)
 

searx/data/core.py

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=missing-module-docstring
+from __future__ import annotations
+
+import pathlib
+
+from searx import logger
+from searx.cache import ExpireCacheCfg, ExpireCacheSQLite
+
+log = logger.getChild("data")
+
+data_dir = pathlib.Path(__file__).parent
+
+_DATA_CACHE: ExpireCacheSQLite = None  # type: ignore
+
+
+def get_cache():
+
+    global _DATA_CACHE  # pylint: disable=global-statement
+
+    if _DATA_CACHE is None:
+        _DATA_CACHE = ExpireCacheSQLite.build_cache(
+            ExpireCacheCfg(
+                name="DATA_CACHE",
+                # MAX_VALUE_LEN=1024 * 200,  # max. 200kB length for a *serialized* value.
+                # MAXHOLD_TIME=60 * 60 * 24 * 7 * 4,  # 4 weeks
+            )
+        )
+    return _DATA_CACHE

searx/data/currencies.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Simple implementation to store currencies data in a SQL database."""
+
+from __future__ import annotations
+
+__all__ = ["CurrenciesDB"]
+
+import json
+import pathlib
+
+from .core import get_cache, log
+
+
+class CurrenciesDB:
+    # pylint: disable=missing-class-docstring
+
+    ctx_names = "data_currencies_names"
+    ctx_iso4217 = "data_currencies_iso4217"
+
+    json_file = pathlib.Path(__file__).parent / "currencies.json"
+
+    def __init__(self):
+        self.cache = get_cache()
+
+    def init(self):
+        if self.cache.properties("currencies loaded") != "OK":
+            self.load()
+            self.cache.properties.set("currencies loaded", "OK")
+        # F I X M E:
+        #     do we need a maintenance .. rember: database is stored
+        #     in /tmp and will be rebuild during the reboot anyway
+
+    def load(self):
+        log.debug("init searx.data.CURRENCIES")
+        with open(self.json_file, encoding="utf-8") as f:
+            data_dict = json.load(f)
+        for key, value in data_dict["names"].items():
+            self.cache.set(key=key, value=value, ctx=self.ctx_names, expire=None)
+        for key, value in data_dict["iso4217"].items():
+            self.cache.set(key=key, value=value, ctx=self.ctx_iso4217, expire=None)
+
+    def name_to_iso4217(self, name):
+        self.init()
+
+        ret_val = self.cache.get(key=name, default=name, ctx=self.ctx_names)
+        if isinstance(ret_val, list):
+            # if more alternatives, use the last in the list
+            ret_val = ret_val[-1]
+        return ret_val
+
+    def iso4217_to_name(self, iso4217, language):
+        self.init()
+
+        iso4217_languages: dict = self.cache.get(key=iso4217, default={}, ctx=self.ctx_names)
+        return iso4217_languages.get(language, iso4217)

searx/engines/currency_convert.py

@@ -3,6 +3,7 @@
 """
 
 import json
+from searx.result_types import EngineResults
 
 # about
 about = {
@@ -28,13 +29,15 @@ def request(_query, params):
     return params
 
 
-def response(resp):
+def response(resp) -> EngineResults:
+    res = EngineResults()
+
     # remove first and last lines to get only json
     json_resp = resp.text[resp.text.find('\n') + 1 : resp.text.rfind('\n') - 2]
     try:
         conversion_rate = float(json.loads(json_resp)["to"][0]["mid"])
     except IndexError:
-        return []
+        return res
     answer = '{0} {1} = {2} {3}, 1 {1} ({5}) = {4} {3} ({6})'.format(
         resp.search_params['amount'],
         resp.search_params['from'],
@@ -46,5 +49,5 @@ def response(resp):
     )
 
     url = f"https://duckduckgo.com/?q={resp.search_params['from']}+to+{resp.search_params['to']}"
-
-    return [{"answer": answer, "url": url}]
+    res.add(res.types.Answer(answer=answer, url=url))
+    return res

searx/search/processors/online_currency.py

@@ -12,24 +12,13 @@ from .online import OnlineProcessor
 parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
 
 
-def normalize_name(name):
+def normalize_name(name: str):
+    name = name.strip()
     name = name.lower().replace('-', ' ').rstrip('s')
     name = re.sub(' +', ' ', name)
     return unicodedata.normalize('NFKD', name).lower()
 
 
-def name_to_iso4217(name):
-    name = normalize_name(name)
-    currency = CURRENCIES['names'].get(name, [name])
-    if isinstance(currency, str):
-        return currency
-    return currency[-1]
-
-
-def iso4217_to_name(iso4217, language):
-    return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217)
-
-
 class OnlineCurrencyProcessor(OnlineProcessor):
     """Processor class used by ``online_currency`` engines."""
 
@@ -52,14 +41,15 @@ class OnlineCurrencyProcessor(OnlineProcessor):
             amount = float(amount_str)
         except ValueError:
             return None
-        from_currency = name_to_iso4217(from_currency.strip())
-        to_currency = name_to_iso4217(to_currency.strip())
+
+        from_currency = CURRENCIES.name_to_iso4217(normalize_name(from_currency))
+        to_currency = CURRENCIES.name_to_iso4217(normalize_name(to_currency))
 
         params['amount'] = amount
         params['from'] = from_currency
         params['to'] = to_currency
-        params['from_name'] = iso4217_to_name(from_currency, 'en')
-        params['to_name'] = iso4217_to_name(to_currency, 'en')
+        params['from_name'] = CURRENCIES.iso4217_to_name(from_currency, "en")
+        params['to_name'] = CURRENCIES.iso4217_to_name(to_currency, "en")
         return params
 
     def get_default_tests(self):

searx/translations/et/LC_MESSAGES/messages.po

@@ -18,18 +18,19 @@
 # Priit Jõerüüt <jrtcdbrg@noreply.codeberg.org>, 2025.
 msgid ""
 msgstr ""
-"Project-Id-Version:  searx\n"
+"Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2025-05-13 19:13+0000\n"
-"PO-Revision-Date: 2025-03-30 01:58+0000\n"
+"PO-Revision-Date: 2025-05-23 07:10+0000\n"
 "Last-Translator: Priit Jõerüüt <jrtcdbrg@noreply.codeberg.org>\n"
+"Language-Team: Estonian <https://translate.codeberg.org/projects/searxng/"
+"searxng/et/>\n"
 "Language: et\n"
-"Language-Team: Estonian "
-"<https://translate.codeberg.org/projects/searxng/searxng/et/>\n"
-"Plural-Forms: nplurals=2; plural=n != 1;\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+"X-Generator: Weblate 5.11.4\n"
 "Generated-By: Babel 2.17.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -685,7 +686,7 @@ msgstr "Avalikud serverid"
 
 #: searx/templates/simple/base.html:75
 msgid "Privacy policy"
-msgstr "Privaatsuspoliitika"
+msgstr "Andmekaitsepõhimõtted"
 
 #: searx/templates/simple/base.html:78
 msgid "Contact instance maintainer"
@@ -2074,4 +2075,3 @@ msgstr "peida video"
 
 #~ msgid "proxied"
 #~ msgstr "proksiserveris"
-

searx/translations/th/LC_MESSAGES/messages.po

@@ -13,20 +13,22 @@
 # Anonymous <anonymous@users.noreply.translate.codeberg.org>, 2025.
 # yuttct <yuttct@users.noreply.translate.codeberg.org>, 2025.
 # return42 <return42@noreply.codeberg.org>, 2025.
+# wetinee <wetinee@noreply.codeberg.org>, 2025.
 msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2025-05-13 19:13+0000\n"
-"PO-Revision-Date: 2025-04-15 10:37+0000\n"
-"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
+"PO-Revision-Date: 2025-05-17 15:04+0000\n"
+"Last-Translator: wetinee <wetinee@noreply.codeberg.org>\n"
+"Language-Team: Thai <https://translate.codeberg.org/projects/searxng/searxng/"
+"th/>\n"
 "Language: th\n"
-"Language-Team: Thai "
-"<https://translate.codeberg.org/projects/searxng/searxng/th/>\n"
-"Plural-Forms: nplurals=1; plural=0;\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=1; plural=0;\n"
+"X-Generator: Weblate 5.11.4\n"
 "Generated-By: Babel 2.17.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -445,11 +447,11 @@ msgstr "ทำการสุ่มค่าที่แตกต่างกั
 #: searx/answerers/statistics.py:36
 #, python-brace-format
 msgid "Compute {func} of the arguments"
-msgstr ""
+msgstr "คำนวณ {func} ของอาร์กิวเมนต์"
 
 #: searx/engines/openstreetmap.py:158
 msgid "Show route in map .."
-msgstr ""
+msgstr "แสดงเส้นทางบนแผนที่ .."
 
 #: searx/engines/pdbe.py:96
 #, python-brace-format
@@ -522,15 +524,15 @@ msgstr "คุณภาพไฟล์"
 
 #: searx/plugins/ahmia_filter.py:32
 msgid "Ahmia blacklist"
-msgstr ""
+msgstr "แบล็กลิสต์ Ahmia"
 
 #: searx/plugins/ahmia_filter.py:33
 msgid "Filter out onion results that appear in Ahmia's blacklist."
-msgstr ""
+msgstr "กรองผลการค้นหา onion ที่อยู่ในแบล็กลิสต์ Ahmia"
 
 #: searx/plugins/calculator.py:38
 msgid "Basic Calculator"
-msgstr ""
+msgstr "เครื่องคิดเลขพื้นฐาน"
 
 #: searx/plugins/calculator.py:39
 msgid "Calculate mathematical expressions via the search bar"
@@ -538,7 +540,7 @@ msgstr "คำนวณนิพจน์คณิตศาสตร์ผ่า
 
 #: searx/plugins/hash_plugin.py:34
 msgid "Hash plugin"
-msgstr ""
+msgstr "ปลั๊กอินแฮช"
 
 #: searx/plugins/hash_plugin.py:35
 msgid "Converts strings to different hash digests."
@@ -575,6 +577,8 @@ msgid ""
 "Displays your IP if the query is \"ip\" and your user agent if the query "
 "is \"user-agent\"."
 msgstr ""
+"จะแสดง IP ของคุณหากคำค้นคือ 'ip' และแสดงข้อมูล User Agent ของคุณหากคำค้นคือ 'user-"
+"agent'"
 
 #: searx/plugins/self_info.py:52
 msgid "Your IP is: "
@@ -599,15 +603,15 @@ msgstr ""
 
 #: searx/plugins/tor_check.py:65
 msgid "Could not download the list of Tor exit-nodes from"
-msgstr ""
+msgstr "ไม่สามารถดาวน์โหลดรายการของโหนดทางออกของ Tor จาก"
 
 #: searx/plugins/tor_check.py:72
 msgid "You are using Tor and it looks like you have the external IP address"
-msgstr ""
+msgstr "คุณกำลังใช้ Tor และดูเหมือนว่าคุณมีที่อยู่ IP ภายนอก"
 
 #: searx/plugins/tor_check.py:76
 msgid "You are not using Tor and you have the external IP address"
-msgstr ""
+msgstr "คุณไม่ได้ใช้ Tor และคุณมีที่อยู่ IP ภายนอก"
 
 #: searx/plugins/tracker_url_remover.py:37
 msgid "Tracker URL remover"
@@ -619,7 +623,7 @@ msgstr "ลบอาร์กิวเมนต์ตัวติดตามอ
 
 #: searx/plugins/unit_converter.py:49
 msgid "Unit converter plugin"
-msgstr ""
+msgstr "ปลั๊กอินแปลงหน่วย"
 
 #: searx/plugins/unit_converter.py:50
 msgid "Convert between units"
@@ -931,7 +935,7 @@ msgstr "ตัวอย่าง"
 
 #: searx/templates/simple/answer/translations.html:21
 msgid "Definitions"
-msgstr ""
+msgstr "คำนิยาม"
 
 #: searx/templates/simple/answer/translations.html:30
 msgid "Synonyms"
@@ -1090,7 +1094,7 @@ msgstr "อนุญาต"
 
 #: searx/templates/simple/preferences/answerers.html:5
 msgid "Keywords (first word in query)"
-msgstr ""
+msgstr "คำหลัก (คำแรกของคำค้นหา)"
 
 #: searx/templates/simple/preferences/answerers.html:6
 #: searx/templates/simple/result_templates/packages.html:7
@@ -1164,6 +1168,7 @@ msgid ""
 "A URL containing your preferences. This URL can be used to restore your "
 "settings on a different device."
 msgstr ""
+"ลิงก์ URL ที่บรรจุค่าการตั้งค่าของคุณ ซึ่งสามารถนำไปใช้กู้คืนการตั้งค่าเหล่านั้นบนอุปกรณ์อื่นได้"
 
 #: searx/templates/simple/preferences/cookies.html:46
 msgid "Copy preferences hash"
@@ -1179,7 +1184,7 @@ msgstr "การตั้งค่าแฮช"
 
 #: searx/templates/simple/preferences/doi_resolver.html:1
 msgid "Digital Object Identifier (DOI)"
-msgstr ""
+msgstr "รหัส DOI"
 
 #: searx/templates/simple/preferences/doi_resolver.html:6
 msgid "Open Access DOI resolver"
@@ -1373,23 +1378,23 @@ msgstr "เปลี่ยนภาษาของเค้าโครง"
 
 #: searx/templates/simple/preferences/urlformatting.html:2
 msgid "URL formatting"
-msgstr ""
+msgstr "การจัดรูปแบบ URL"
 
 #: searx/templates/simple/preferences/urlformatting.html:8
 msgid "Pretty"
-msgstr ""
+msgstr "แบบสวยงาม"
 
 #: searx/templates/simple/preferences/urlformatting.html:13
 msgid "Full"
-msgstr ""
+msgstr "แบบเต็ม"
 
 #: searx/templates/simple/preferences/urlformatting.html:18
 msgid "Host"
-msgstr ""
+msgstr "โฮสต์"
 
 #: searx/templates/simple/preferences/urlformatting.html:23
 msgid "Change result URL formatting"
-msgstr ""
+msgstr "เปลี่ยนรูปแบบ URL ผลการค้นหา"
 
 #: searx/templates/simple/result_templates/code.html:13
 msgid "repo"
@@ -1787,4 +1792,3 @@ msgstr "ซ่อนวิดีโอ"
 
 #~ msgid "proxied"
 #~ msgstr "พร็อกซี่"
-

searxng_extra/update/update_currencies.py

@@ -15,9 +15,7 @@ import json
 
 from searx.locales import LOCALE_NAMES, locales_initialize
 from searx.engines import wikidata, set_loggers
-from searx.data import data_dir
-
-DATA_FILE = data_dir / 'currencies.json'
+from searx.data.currencies import CurrenciesDB
 
 set_loggers(wikidata, 'wikidata')
 locales_initialize()
@@ -149,7 +147,7 @@ def main():
         if len(db['names'][name]) == 1:
             db['names'][name] = db['names'][name][0]
 
-    with DATA_FILE.open('w', encoding='utf8') as f:
+    with CurrenciesDB.json_file.open('w', encoding='utf8') as f:
         json.dump(db, f, indent=4, sort_keys=True, ensure_ascii=False)