[fix] sqlitedb: DB connector -- unset isolation_level

A Connection object can be used as a context manager that automatically commits
or rolls back open transactions when leaving the body of the context manager.

If the connection attribute ``isolation_level`` is **not** ``None``, new
transactions are implicitly opened before ``execute()`` and ``executemany()``
executes SQL- INSERT, UPDATE, DELETE, or REPLACE statements [1].

The old implementation set ``isolation_level`` to ``None`` and thereby prevented
the context manager from opening and closing the transaction scopes.

[1] https://docs.python.org/3/library/sqlite3.html#sqlite3-transaction-control-isolation-level
[2] https://github.com/searxng/searxng/pull/5239#discussion_r2381416731

Reported-by: Ivan G <igabaldon@inetol.net> [2]
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/cache.py

@@ -29,6 +29,8 @@ from searx import get_setting
 
 log = logger.getChild("cache")
 
+CacheRowType: typing.TypeAlias = tuple[str, typing.Any, int | None]
+
 
 class ExpireCacheCfg(msgspec.Struct):  # pylint: disable=too-few-public-methods
     """Configuration of a :py:obj:`ExpireCache` cache."""
@@ -81,7 +83,7 @@ class ExpireCacheCfg(msgspec.Struct):  # pylint: disable=too-few-public-methods
 class ExpireCacheStats:
     """Dataclass which provides information on the status of the cache."""
 
-    cached_items: dict[str, list[tuple[str, typing.Any, int]]]
+    cached_items: dict[str, list[CacheRowType]]
     """Values in the cache mapped by context name.
 
     .. code: python
@@ -108,7 +110,9 @@ class ExpireCacheStats:
                 continue
 
             for key, value, expire in kv_list:
-                valid_until = datetime.datetime.fromtimestamp(expire).strftime("%Y-%m-%d %H:%M:%S")
+                valid_until = ""
+                if expire:
+                    valid_until = datetime.datetime.fromtimestamp(expire).strftime("%Y-%m-%d %H:%M:%S")
                 c_kv += 1
                 lines.append(f"[{ctx_name:20s}] {valid_until} {key:12}" f" --> ({type(value).__name__}) {value} ")
 
@@ -339,38 +343,97 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
         exists, it will be created (on demand) by :py:obj:`self.create_table
         <ExpireCacheSQLite.create_table>`.
         """
+        c, err_msg_list = self._setmany([(key, value, expire)], ctx=ctx)
+        if c:
+            log.debug("%s -- %s: key '%s' updated or inserted (%s errors)", self.cfg.name, ctx, key, len(err_msg_list))
+        else:
+            for msg in err_msg_list:
+                log.error("%s -- %s: %s", self.cfg.name, ctx, msg)
+        return bool(c)
+
+    def setmany(
+        self,
+        opt_list: list[CacheRowType],
+        ctx: str | None = None,
+    ) -> int:
+        """Efficient bootload of the cache from a list of options.  The list
+        contains tuples with the arguments described in
+        :py:obj:`ExpireCacheSQLite.set`."""
+        _start = time.time()
+        c, err_msg_list = self._setmany(opt_list=opt_list, ctx=ctx)
+        _end = time.time()
+        for msg in err_msg_list:
+            log.error("%s -- %s: %s", self.cfg.name, ctx, msg)
+
+        log.debug(
+            "%s -- %s: %s/%s key/value pairs updated or inserted in %s sec (%s errors)",
+            self.cfg.name,
+            ctx,
+            c,
+            len(opt_list),
+            _end - _start,
+            len(err_msg_list),
+        )
+        return c
+
+    def _setmany(
+        self,
+        opt_list: list[CacheRowType],
+        ctx: str | None = None,
+    ) -> tuple[int, list[str]]:
+
         table = ctx
         self.maintenance()
 
-        value = self.serialize(value=value)
-        if len(value) > self.cfg.MAX_VALUE_LEN:
-            log.warning("ExpireCache.set(): %s.key='%s' - value too big to cache (len: %s)  ", table, value, len(value))
-            return False
-
-        if not expire:
-            expire = self.cfg.MAXHOLD_TIME
-        expire = int(time.time()) + expire
-
         table_name = table
         if not table_name:
             table_name = self.normalize_name(self.cfg.name)
         self.create_table(table_name)
 
-        sql = (
+        sql_str = (
             f"INSERT INTO {table_name} (key, value, expire) VALUES (?, ?, ?)"
             f"    ON CONFLICT DO "
             f"UPDATE SET value=?, expire=?"
         )
 
+        sql_rows: list[
+            tuple[
+                str,  # key
+                typing.Any,  # value
+                int | None,  # expire
+                typing.Any,  # value
+                int | None,  # expire
+            ]
+        ] = []
+
+        err_msg_list: list[str] = []
+        for key, _val, expire in opt_list:
+
+            value: bytes = self.serialize(value=_val)
+            if len(value) > self.cfg.MAX_VALUE_LEN:
+                err_msg_list.append(f"{table}.key='{key}' - serialized value too big to cache (len: {len(value)}) ")
+                continue
+
+            if not expire:
+                expire = self.cfg.MAXHOLD_TIME
+            expire = int(time.time()) + expire
+
+            # positional arguments of the INSERT INTO statement
+            sql_args = (key, value, expire, value, expire)
+            sql_rows.append(sql_args)
+
+        if not sql_rows:
+            return 0, err_msg_list
+
         if table:
             with self.DB:
-                self.DB.execute(sql, (key, value, expire, value, expire))
+                self.DB.executemany(sql_str, sql_rows)
         else:
             with self.connect() as conn:
-                conn.execute(sql, (key, value, expire, value, expire))
+                conn.executemany(sql_str, sql_rows)
             conn.close()
 
-        return True
+        return len(sql_rows), err_msg_list
 
     def get(self, key: str, default: typing.Any = None, ctx: str | None = None) -> typing.Any:
         """Get value of ``key`` from table given by argument ``ctx``.  If
@@ -410,7 +473,7 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
                 yield row[0], self.deserialize(row[1])
 
     def state(self) -> ExpireCacheStats:
-        cached_items: dict[str, list[tuple[str, typing.Any, int]]] = {}
+        cached_items: dict[str, list[CacheRowType]] = {}
         for table in self.table_names:
             cached_items[table] = []
             for row in self.DB.execute(f"SELECT key, value, expire FROM {table}"):

searx/data/__main__.py

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Command line implementation"""
+
+import typer
+
+from .core import get_cache
+
+app = typer.Typer()
+
+
+@app.command()
+def state():
+    """show state of the cache"""
+    cache = get_cache()
+    for table in cache.table_names:
+        for row in cache.DB.execute(f"SELECT count(*) FROM {table}"):
+            print(f"cache table {table} holds {row[0]} key/value pairs")
+
+
+app()

searx/data/currencies.py

@@ -9,6 +9,9 @@ import pathlib
 
 from .core import get_cache, log
 
+if t.TYPE_CHECKING:
+    from searx.cache import CacheRowType
+
 
 @t.final
 class CurrenciesDB:
@@ -35,10 +38,11 @@ class CurrenciesDB:
         log.debug("init searx.data.CURRENCIES")
         with open(self.json_file, encoding="utf-8") as f:
             data_dict: dict[str, dict[str, str]] = json.load(f)
-        for key, value in data_dict["names"].items():
-            self.cache.set(key=key, value=value, ctx=self.ctx_names, expire=None)
-        for key, value in data_dict["iso4217"].items():
-            self.cache.set(key=key, value=value, ctx=self.ctx_iso4217, expire=None)
+
+        rows: "list[CacheRowType]" = [(k, v, None) for k, v in data_dict["names"].items()]
+        self.cache.setmany(rows, ctx=self.ctx_names)
+        rows = [(k, v, None) for k, v in data_dict["iso4217"].items()]
+        self.cache.setmany(rows, ctx=self.ctx_iso4217)
 
     def name_to_iso4217(self, name: str) -> str | None:
         self.init()

searx/data/tracker_patterns.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Simple implementation to store TrackerPatterns data in a SQL database."""
 
-import typing
+import typing as t
 
 __all__ = ["TrackerPatternsDB"]
 
@@ -14,9 +14,14 @@ from httpx import HTTPError
 from searx.data.core import get_cache, log
 from searx.network import get as http_get
 
+if t.TYPE_CHECKING:
+    from searx.cache import CacheRowType
+
+
 RuleType = tuple[str, list[str], list[str]]
 
 
+@t.final
 class TrackerPatternsDB:
     # pylint: disable=missing-class-docstring
 
@@ -31,9 +36,9 @@ class TrackerPatternsDB:
 
     class Fields:
         # pylint: disable=too-few-public-methods, invalid-name
-        url_regexp: typing.Final = 0  # URL (regular expression) match condition of the link
-        url_ignore: typing.Final = 1  # URL (regular expression) to ignore
-        del_args: typing.Final = 2  # list of URL arguments (regular expression) to delete
+        url_regexp: t.Final = 0  # URL (regular expression) match condition of the link
+        url_ignore: t.Final = 1  # URL (regular expression) to ignore
+        del_args: t.Final = 2  # list of URL arguments (regular expression) to delete
 
     def __init__(self):
         self.cache = get_cache()
@@ -49,19 +54,25 @@ class TrackerPatternsDB:
 
     def load(self):
         log.debug("init searx.data.TRACKER_PATTERNS")
-        for rule in self.iter_clear_list():
-            self.add(rule)
+        rows: "list[CacheRowType]" = []
 
-    def add(self, rule: RuleType):
-        self.cache.set(
-            key=rule[self.Fields.url_regexp],
-            value=(
+        for rule in self.iter_clear_list():
+            key = rule[self.Fields.url_regexp]
+            value = (
                 rule[self.Fields.url_ignore],
                 rule[self.Fields.del_args],
-            ),
-            ctx=self.ctx_name,
-            expire=None,
+            )
+            rows.append((key, value, None))
+
+        self.cache.setmany(rows, ctx=self.ctx_name)
+
+    def add(self, rule: RuleType):
+        key = rule[self.Fields.url_regexp]
+        value = (
+            rule[self.Fields.url_ignore],
+            rule[self.Fields.del_args],
         )
+        self.cache.set(key=key, value=value, ctx=self.ctx_name, expire=None)
 
     def rules(self) -> Iterator[RuleType]:
         self.init()

searx/engines/openstreetmap.py

@@ -446,7 +446,7 @@ def get_key_label(key_name, lang):
         # https://taginfo.openstreetmap.org/keys/currency#values
         currency = key_name.split(':')
         if len(currency) > 1:
-            o = CURRENCIES['iso4217'].get(currency[1])
+            o = CURRENCIES.iso4217_to_name(currency[1], lang)
             if o:
                 return get_label(o, lang).lower()
             return currency[1]

searx/search/processors/online_currency.py

@@ -52,10 +52,6 @@ class OnlineCurrencyProcessor(OnlineProcessor):
 
     engine_type: str = "online_currency"
 
-    def initialize(self, callback: t.Callable[["EngineProcessor", bool], bool]):
-        CURRENCIES.init()
-        super().initialize(callback)
-
     def get_params(self, search_query: "SearchQuery", engine_category: str) -> OnlineCurrenciesParams | None:
         """Returns a dictionary with the :ref:`request params <engine request
         online_currency>` (:py:obj:`OnlineCurrenciesParams`).  ``None`` is

searx/sqlitedb.py

@@ -127,14 +127,19 @@ class SQLiteAppl(abc.ABC):
         "check_same_thread": bool(SQLITE_THREADING_MODE != "serialized"),
         "cached_statements": 0,  # https://github.com/python/cpython/issues/118172
         # "uri": False,
-        "isolation_level": None,
+        # "isolation_level": "",
+        # "autocommit": sqlite3.LEGACY_TRANSACTION_CONTROL,
     }  # fmt:skip
     """Connection arguments (:py:obj:`sqlite3.connect`)
 
-    ``check_same_thread``:
+    ``check_same_thread``: *bool*
       Is disabled by default when :py:obj:`SQLITE_THREADING_MODE` is
-      ``serialized``.  The check is more of a hindrance in this case because it
-      would prevent a DB connector from being used in multiple threads.
+      `serialized`.  The check is more of a hindrance when threadsafety_ is
+      `serialized` because it would prevent a DB connector from being used in
+      multiple threads.
+
+      Is enabled when threadsafety_ is ``single-thread`` or ``multi-thread``
+      (when threads cannot share a connection PEP-0249_).
 
     ``cached_statements``:
       Is set to ``0`` by default.  Note: Python 3.12+ fetch result are not
@@ -149,6 +154,40 @@ class SQLiteAppl(abc.ABC):
 
       The workaround for SQLite3 multithreading cache inconsistency is to set
       option ``cached_statements`` to ``0`` by default.
+
+    ``isolation_level``: *unset*
+      If the connection attribute isolation_level_ is **not** ``None``, new
+      transactions are implicitly opened before ``execute()`` and
+      ``executemany()`` executes SQL- INSERT, UPDATE, DELETE, or REPLACE
+      statements `[1]`_.
+
+      By default, the value is not set, which means the default from Python is
+      used: Python's default is ``""``, which is an alias for ``"DEFERRED"``.
+
+    ``autocommit``: *unset*
+      Starting with Python 3.12 the DB connection has a ``autocommit`` attribute
+      and the recommended way of controlling transaction behaviour is through
+      this attribute `[2]`_.
+
+      By default, the value is not set, which means the default from Python is
+      used: Python's default is the constant LEGACY_TRANSACTION_CONTROL_:
+      Pre-Python 3.12 (non-PEP 249-compliant) transaction control, see
+      ``isolation_level`` above for more details.
+
+    .. _PEP-0249:
+        https://peps.python.org/pep-0249/#threadsafety
+    .. _threadsafety:
+        https://docs.python.org/3/library/sqlite3.html#sqlite3.threadsafety
+    .. _isolation_level:
+        https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.isolation_level
+    .. _[1]:
+        https://docs.python.org/3/library/sqlite3.html#sqlite3-transaction-control-isolation-level
+    .. _autocommit:
+        https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.autocommit
+    .. _[2]:
+        https://docs.python.org/3/library/sqlite3.html#transaction-control-via-the-autocommit-attribute
+    .. _LEGACY_TRANSACTION_CONTROL:
+        https://docs.python.org/3/library/sqlite3.html#sqlite3.LEGACY_TRANSACTION_CONTROL
     """
 
     def __init__(self, db_url: str):