[fix] cache.ExpireCache: definition of a context name for the key

The definition of a context name belongs in the abstract base class (was
previously only in the concrete implementation for the SQLite adapter).

Suggested-by: @dalf https://github.com/searxng/searxng/pull/4650#discussion_r2069873853
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

docs/admin/settings/settings_server.rst

@@ -29,6 +29,8 @@
   directly using ``python searx/webapp.py``.  Doesn't apply to a SearXNG
   services running behind a proxy and using socket communications.
 
+.. _server.secret_key:
+
 ``secret_key`` : ``$SEARXNG_SECRET``
   Used for cryptography purpose.
 

docs/dev/engines/enginelib.rst

@@ -4,17 +4,11 @@
 Engine Library
 ==============
 
-.. contents::
-   :depth: 2
-   :local:
-   :backlinks: entry
-
 .. automodule:: searx.enginelib
    :members:
 
 .. _searx.enginelib.traits:
 
-
 Engine traits
 =============
 

docs/dev/engines/offline_concept.rst

@@ -29,7 +29,7 @@ Programming Interface
   parameter. This function can be omitted, if there is no need to setup anything
   in advance.
 
-:py:func:`search(query, params) <searx.engines.demo_offline.searc>`
+:py:func:`search(query, params) <searx.engines.demo_offline.search>`
   Each offline engine has a function named ``search``.  This function is
   responsible to perform a search and return the results in a presentable
   format. (Where *presentable* means presentable by the selected result

docs/src/searx.cache.rst

@@ -0,0 +1,8 @@
+.. _searx.cache:
+
+======
+Caches
+======
+
+.. automodule:: searx.cache
+   :members:

searx/cache.py

@@ -0,0 +1,405 @@
+"""Implementation of caching solutions.
+
+- :py:obj:`searx.cache.ExpireCache` and its :py:obj:`searx.cache.ExpireCacheCfg`
+
+----
+"""
+
+from __future__ import annotations
+
+__all__ = ["ExpireCacheCfg", "ExpireCacheStats", "ExpireCache", "ExpireCacheSQLite"]
+
+import abc
+import dataclasses
+import datetime
+import hashlib
+import hmac
+import os
+import pickle
+import sqlite3
+import string
+import tempfile
+import time
+import typing
+
+import msgspec
+
+from searx import sqlitedb
+from searx import logger
+from searx import get_setting
+
+log = logger.getChild("cache")
+
+
+class ExpireCacheCfg(msgspec.Struct):  # pylint: disable=too-few-public-methods
+    """Configuration of a :py:obj:`ExpireCache` cache."""
+
+    name: str
+    """Name of the cache."""
+
+    db_url: str = ""
+    """URL of the SQLite DB, the path to the database file.  If unset a default
+    DB will be created in `/tmp/sxng_cache_{self.name}.db`"""
+
+    MAX_VALUE_LEN: int = 1024 * 10
+    """Max lenght of a *serialized* value."""
+
+    MAXHOLD_TIME: int = 60 * 60 * 24 * 7  # 7 days
+    """Hold time (default in sec.), after which a value is removed from the cache."""
+
+    MAINTENANCE_PERIOD: int = 60 * 60  # 2h
+    """Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to
+    ``auto``."""
+
+    MAINTENANCE_MODE: typing.Literal["auto", "off"] = "auto"
+    """Type of maintenance mode
+
+    ``auto``:
+      Maintenance is carried out automatically as part of the maintenance
+      intervals (:py:obj:`MAINTENANCE_PERIOD`); no external process is required.
+
+    ``off``:
+      Maintenance is switched off and must be carried out by an external process
+      if required.
+    """
+
+    password: bytes = get_setting("server.secret_key").encode()  # type: ignore
+    """Password used by :py:obj:`ExpireCache.secret_hash`.
+
+    The default password is taken from :ref:`secret_key <server.secret_key>`.
+    When the password is changed, the hashed keys in the cache can no longer be
+    used, which is why all values in the cache are deleted when the password is
+    changed.
+    """
+
+    def __post_init__(self):
+        # if db_url is unset, use a default DB in /tmp/sxng_cache_{name}.db
+        if not self.db_url:
+            self.db_url = tempfile.gettempdir() + os.sep + f"sxng_cache_{ExpireCache.normalize_name(self.name)}.db"
+
+
+@dataclasses.dataclass
+class ExpireCacheStats:
+    """Dataclass wich provides information on the status of the cache."""
+
+    cached_items: dict[str, list[tuple[str, typing.Any, int]]]
+    """Values in the cache mapped by context name.
+
+    .. code: python
+
+       {
+           "context name": [
+               ("foo key": "foo value", <expire>),
+               ("bar key": "bar value", <expire>),
+               # ...
+           ],
+           # ...
+       }
+    """
+
+    def report(self):
+        c_ctx = 0
+        c_kv = 0
+        lines = []
+
+        for ctx_name, kv_list in self.cached_items.items():
+            c_ctx += 1
+            if not kv_list:
+                lines.append(f"[{ctx_name:20s}] empty")
+                continue
+
+            for key, value, expire in kv_list:
+                valid_until = datetime.datetime.fromtimestamp(expire).strftime("%Y-%m-%d %H:%M:%S")
+                c_kv += 1
+                lines.append(f"[{ctx_name:20s}] {valid_until} {key:12}" f" --> ({type(value).__name__}) {value} ")
+
+        lines.append(f"Number of contexts: {c_ctx}")
+        lines.append(f"number of key/value pairs: {c_kv}")
+        return "\n".join(lines)
+
+
+class ExpireCache(abc.ABC):
+    """Abstract base class for the implementation of a key/value cache
+    with expire date."""
+
+    cfg: ExpireCacheCfg
+
+    hash_token = "hash_token"
+
+    @abc.abstractmethod
+    def set(self, key: str, value: typing.Any, expire: int | None, ctx: str | None = None) -> bool:
+        """Set *key* to *value*.  To set a timeout on key use argument
+        ``expire`` (in sec.).  If expire is unset the default is taken from
+        :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`.  After the timeout has expired,
+        the key will automatically be deleted.
+
+        The ``ctx`` argument specifies the context of the ``key``.  A key is
+        only unique in its context.
+
+        The concrete implementations of this abstraction determine how the
+        context is mapped in the connected database.  In SQL databases, for
+        example, the context is a DB table or in a Key/Value DB it could be
+        a prefix for the key.
+
+        If the context is not specified (the default is ``None``) then a
+        default context should be used, e.g. a default table for SQL databases
+        or a default prefix in a Key/Value DB.
+        """
+
+    @abc.abstractmethod
+    def get(self, key: str, default=None, ctx: str | None = None) -> typing.Any:
+        """Return *value* of *key*.  If key is unset, ``None`` is returned."""
+
+    @abc.abstractmethod
+    def maintenance(self, force: bool = False, truncate: bool = False) -> bool:
+        """Performs maintenance on the cache.
+
+        ``force``:
+          Maintenance should be carried out even if the maintenance interval has
+          not yet been reached.
+
+        ``truncate``:
+          Truncate the entire cache, which is necessary, for example, if the
+          password has changed.
+        """
+
+    @abc.abstractmethod
+    def state(self) -> ExpireCacheStats:
+        """Returns a :py:obj:`ExpireCacheStats`, which provides information
+        about the status of the cache."""
+
+    @staticmethod
+    def build_cache(cfg: ExpireCacheCfg) -> ExpireCache:
+        """Factory to build a caching instance.
+
+        .. note::
+
+           Currently, only the SQLite adapter is available, but other database
+           types could be implemented in the future, e.g. a Valkey (Redis)
+           adapter.
+        """
+        return ExpireCacheSQLite(cfg)
+
+    @staticmethod
+    def normalize_name(name: str) -> str:
+        """Returns a normalized name that can be used as a file name or as a SQL
+        table name (is used, for example, to normalize the context name)."""
+
+        _valid = "-_." + string.ascii_letters + string.digits
+        return "".join([c for c in name if c in _valid])
+
+    def serialize(self, value: typing.Any) -> bytes:
+        dump: bytes = pickle.dumps(value)
+        return dump
+
+    def deserialize(self, value: bytes) -> typing.Any:
+        obj = pickle.loads(value)
+        return obj
+
+    def secret_hash(self, name: str | bytes) -> str:
+        """Creates a hash of the argument ``name``.  The hash value is formed
+        from the ``name`` combined with the :py:obj:`password
+        <ExpireCacheCfg.password>`.  Can be used, for example, to make the
+        ``key`` stored in the DB unreadable for third parties."""
+
+        if isinstance(name, str):
+            name = bytes(name, encoding='utf-8')
+        m = hmac.new(name + self.cfg.password, digestmod='sha256')
+        return m.hexdigest()
+
+
+class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
+    """Cache that manages key/value pairs in a SQLite DB.  The DB model in the
+    SQLite DB is implemented in abstract class :py:obj:`SQLiteAppl
+    <searx.sqlitedb.SQLiteAppl>`.
+
+    The following configurations are required / supported:
+
+    - :py:obj:`ExpireCacheCfg.db_url`
+    - :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`
+    - :py:obj:`ExpireCacheCfg.MAINTENANCE_PERIOD`
+    - :py:obj:`ExpireCacheCfg.MAINTENANCE_MODE`
+    """
+
+    DB_SCHEMA = 1
+
+    # The key/value tables will be created on demand by self.create_table
+    DDL_CREATE_TABLES = {}
+
+    CACHE_TABLE_PREFIX = "CACHE-TABLE-"
+
+    def __init__(self, cfg: ExpireCacheCfg):
+        """An instance of the SQLite expire cache is build up from a
+        :py:obj:`config <ExpireCacheCfg>`."""
+
+        self.cfg = cfg
+        if cfg.db_url == ":memory:":
+            log.critical("don't use SQLite DB in :memory: in production!!")
+        super().__init__(cfg.db_url)
+
+    def init(self, conn: sqlite3.Connection) -> bool:
+        ret_val = super().init(conn)
+        if not ret_val:
+            return False
+
+        new = hashlib.sha256(self.cfg.password).hexdigest()
+        old = self.properties(self.hash_token)
+        if old != new:
+            if old is not None:
+                log.warning("[%s] hash token changed: truncate all cache tables", self.cfg.name)
+            self.maintenance(force=True, truncate=True)
+            self.properties.set(self.hash_token, new)
+
+        return True
+
+    def maintenance(self, force: bool = False, truncate: bool = False) -> bool:
+
+        if not force and int(time.time()) < self.next_maintenance_time:
+            # log.debug("no maintenance required yet, next maintenance interval is in the future")
+            return False
+
+        # Prevent parallel DB maintenance cycles from other DB connections
+        # (e.g. in multi thread or process environments).
+        self.properties.set("LAST_MAINTENANCE", "")  # hint: this (also) sets the m_time of the property!
+
+        if truncate:
+            self.truncate_tables(self.table_names)
+            return True
+
+        # drop items by expire time stamp ..
+        expire = int(time.time())
+
+        with self.connect() as conn:
+            for table in self.table_names:
+                res = conn.execute(f"DELETE FROM {table} WHERE expire < ?", (expire,))
+                log.debug("deleted %s keys from table %s (expire date reached)", res.rowcount, table)
+
+        # Vacuuming the WALs
+        # https://www.theunterminatedstring.com/sqlite-vacuuming/
+
+        conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+        conn.close()
+
+        return True
+
+    def create_table(self, table: str) -> bool:
+        """Create DB ``table`` if it has not yet been created, no recreates are
+        initiated if the table already exists.
+        """
+        if table in self.table_names:
+            # log.debug("key/value table %s exists in DB (no need to recreate)", table)
+            return False
+
+        log.info("key/value table '%s' NOT exists in DB -> create DB table ..", table)
+        sql_table = "\n".join(
+            [
+                f"CREATE TABLE IF NOT EXISTS {table} (",
+                "  key        TEXT,",
+                "  value      BLOB,",
+                f"  expire     INTEGER DEFAULT (strftime('%s', 'now') + {self.cfg.MAXHOLD_TIME}),",
+                "PRIMARY KEY (key))",
+            ]
+        )
+        sql_index = f"CREATE INDEX IF NOT EXISTS index_expire_{table} ON {table}(expire);"
+        with self.connect() as conn:
+            conn.execute(sql_table)
+            conn.execute(sql_index)
+        conn.close()
+
+        self.properties.set(f"{self.CACHE_TABLE_PREFIX}-{table}", table)
+        return True
+
+    @property
+    def table_names(self) -> list[str]:
+        """List of key/value tables already created in the DB."""
+        sql = f"SELECT value FROM properties WHERE name LIKE '{self.CACHE_TABLE_PREFIX}%%'"
+        rows = self.DB.execute(sql).fetchall() or []
+        return [r[0] for r in rows]
+
+    def truncate_tables(self, table_names: list[str]):
+        log.debug("truncate table: %s", ",".join(table_names))
+        with self.connect() as conn:
+            for table in table_names:
+                conn.execute(f"DELETE FROM {table}")
+        conn.close()
+        return True
+
+    @property
+    def next_maintenance_time(self) -> int:
+        """Returns (unix epoch) time of the next maintenance."""
+
+        return self.cfg.MAINTENANCE_PERIOD + self.properties.m_time("LAST_MAINTENANCE", int(time.time()))
+
+    # implement ABC methods of ExpireCache
+
+    def set(self, key: str, value: typing.Any, expire: int | None, ctx: str | None = None) -> bool:
+        """Set key/value in DB table given by argument ``ctx``.  If expire is
+        unset the default is taken from :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`.
+        If ``ctx`` argument is ``None`` (the default), a table name is
+        generated from the :py:obj:`ExpireCacheCfg.name`.  If DB table does not
+        exists, it will be created (on demand) by :py:obj:`self.create_table
+        <ExpireCacheSQLite.create_table>`.
+        """
+        table = ctx
+        self.maintenance()
+
+        value = self.serialize(value=value)
+        if len(value) > self.cfg.MAX_VALUE_LEN:
+            log.warning("ExpireCache.set(): %s.key='%s' - value too big to cache (len: %s)  ", table, value, len(value))
+            return False
+
+        if not expire:
+            expire = self.cfg.MAXHOLD_TIME
+        expire = int(time.time()) + expire
+
+        table_name = table
+        if not table_name:
+            table_name = self.normalize_name(self.cfg.name)
+        self.create_table(table_name)
+
+        sql = (
+            f"INSERT INTO {table_name} (key, value, expire) VALUES (?, ?, ?)"
+            f"    ON CONFLICT DO "
+            f"UPDATE SET value=?, expire=?"
+        )
+
+        if table:
+            with self.DB:
+                self.DB.execute(sql, (key, value, expire, value, expire))
+        else:
+            with self.connect() as conn:
+                conn.execute(sql, (key, value, expire, value, expire))
+            conn.close()
+
+        return True
+
+    def get(self, key: str, default=None, ctx: str | None = None) -> typing.Any:
+        """Get value of ``key`` from table given by argument ``ctx``.  If
+        ``ctx`` argument is ``None`` (the default), a table name is generated
+        from the :py:obj:`ExpireCacheCfg.name`.  If ``key`` not exists (in
+        table), the ``default`` value is returned.
+
+        """
+        table = ctx
+        self.maintenance()
+
+        if not table:
+            table = self.normalize_name(self.cfg.name)
+
+        if table not in self.table_names:
+            return default
+
+        sql = f"SELECT value FROM {table} WHERE key = ?"
+        row = self.DB.execute(sql, (key,)).fetchone()
+        if row is None:
+            return default
+
+        return self.deserialize(row[0])
+
+    def state(self) -> ExpireCacheStats:
+        cached_items = {}
+        for table in self.table_names:
+            cached_items[table] = []
+            for row in self.DB.execute(f"SELECT key, value, expire FROM {table}"):
+                cached_items[table].append((row[0], self.deserialize(row[1]), row[2]))
+        return ExpireCacheStats(cached_items=cached_items)

searx/enginelib/__init__.py

@@ -1,6 +1,16 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Implementations of the framework for the SearXNG engines.
 
+- :py:obj:`searx.enginelib.EngineCache`
+- :py:obj:`searx.enginelib.Engine`
+- :py:obj:`searx.enginelib.traits`
+
+There is a command line for developer purposes and for deeper analysis.  Here is
+an example in which the command line is called in the development environment::
+
+  $ ./manage pyenv.cmd bash --norc --noprofile
+  (py3) python -m searx.enginelib --help
+
 .. hint::
 
    The long term goal is to modularize all implementations of the engine
@@ -9,16 +19,158 @@
    - move implementations of the :ref:`searx.engines loader` to a new module in
      the :py:obj:`searx.enginelib` namespace.
 
+-----
+
 """
-
-
 from __future__ import annotations
-from typing import List, Callable, TYPE_CHECKING
+
+__all__ = ["EngineCache", "Engine", "ENGINES_CACHE"]
+
+from typing import List, Callable, TYPE_CHECKING, Any
+import string
+import typer
+
+from ..cache import ExpireCache, ExpireCacheCfg
 
 if TYPE_CHECKING:
     from searx.enginelib import traits
 
 
+ENGINES_CACHE = ExpireCache.build_cache(
+    ExpireCacheCfg(
+        name="ENGINES_CACHE",
+        MAXHOLD_TIME=60 * 60 * 24 * 7,  # 7 days
+        MAINTENANCE_PERIOD=60 * 60,  # 2h
+    )
+)
+"""Global :py:obj:`searx.cache.ExpireCacheSQLite` instance where the cached
+values from all engines are stored.  The `MAXHOLD_TIME` is 7 days and the
+`MAINTENANCE_PERIOD` is set to two hours."""
+
+app = typer.Typer()
+
+
+@app.command()
+def state():
+    """Show state for the caches of the engines."""
+
+    title = "cache tables and key/values"
+    print(title)
+    print("=" * len(title))
+    print(ENGINES_CACHE.state().report())
+    print()
+    title = f"properties of {ENGINES_CACHE.cfg.name}"
+    print(title)
+    print("=" * len(title))
+    print(str(ENGINES_CACHE.properties))  # type: ignore
+
+
+@app.command()
+def maintenance(force: bool = True):
+    """Carry out maintenance on cache of the engines."""
+    ENGINES_CACHE.maintenance(force=force)
+
+
+class EngineCache:
+    """Persistent (SQLite) key/value cache that deletes its values again after
+    ``expire`` seconds (default/max: :py:obj:`MAXHOLD_TIME
+    <searx.cache.ExpireCacheCfg.MAXHOLD_TIME>`).  This class is a wrapper around
+    :py:obj:`ENGINES_CACHE` (:py:obj:`ExpireCacheSQLite
+    <searx.cache.ExpireCacheSQLite>`).
+
+    In the :origin:`searx/engines/demo_offline.py` engine you can find an
+    exemplary implementation of such a cache other exaples are implemeted
+    in:
+
+    - :origin:`searx/engines/radio_browser.py`
+    - :origin:`searx/engines/soundcloud.py`
+    - :origin:`searx/engines/startpage.py`
+
+    .. code: python
+
+       from searx.enginelib import EngineCache
+       CACHE: EngineCache
+
+       def init(engine_settings):
+           global CACHE
+           CACHE = EngineCache(engine_settings["name"])
+
+       def request(query, params):
+           token = CACHE.get(key="token")
+           if token is None:
+               token = get_token()
+               # cache token of this engine for 1h
+               CACHE.set(key="token", value=token, expire=3600)
+           ...
+
+    For introspection of the DB, jump into developer environment and run command to
+    show cache state::
+
+        $ ./manage pyenv.cmd bash --norc --noprofile
+        (py3) python -m searx.enginelib cache state
+
+        cache tables and key/values
+        ===========================
+        [demo_offline        ] 2025-04-22 11:32:50 count        --> (int) 4
+        [startpage           ] 2025-04-22 12:32:30 SC_CODE      --> (str) fSOBnhEMlDfE20
+        [duckduckgo          ] 2025-04-22 12:32:31 4dff493e.... --> (str) 4-128634958369380006627592672385352473325
+        [duckduckgo          ] 2025-04-22 12:40:06 3e2583e2.... --> (str) 4-263126175288871260472289814259666848451
+        [radio_browser       ] 2025-04-23 11:33:08 servers      --> (list) ['https://de2.api.radio-browser.info',  ...]
+        [soundcloud          ] 2025-04-29 11:40:06 guest_client_id --> (str) EjkRJG0BLNEZquRiPZYdNtJdyGtTuHdp
+        [wolframalpha        ] 2025-04-22 12:40:06 code         --> (str) 5aa79f86205ad26188e0e26e28fb7ae7
+        number of tables: 6
+        number of key/value pairs: 7
+
+    In the "cache tables and key/values" section, the table name (engine name) is at
+    first position on the second there is the calculated expire date and on the
+    third and fourth position the key/value is shown.
+
+    About duckduckgo: The *vqd coode* of ddg depends on the query term and therefore
+    the key is a hash value of the query term (to not to store the raw query term).
+
+    In the "properties of ENGINES_CACHE" section all properties of the SQLiteAppl /
+    ExpireCache and their last modification date are shown::
+
+        properties of ENGINES_CACHE
+        ===========================
+        [last modified: 2025-04-22 11:32:27] DB_SCHEMA           : 1
+        [last modified: 2025-04-22 11:32:27] LAST_MAINTENANCE    :
+        [last modified: 2025-04-22 11:32:27] crypt_hash          : ca612e3566fdfd7cf7efe...
+        [last modified: 2025-04-22 11:32:30] CACHE-TABLE--demo_offline: demo_offline
+        [last modified: 2025-04-22 11:32:30] CACHE-TABLE--startpage: startpage
+        [last modified: 2025-04-22 11:32:31] CACHE-TABLE--duckduckgo: duckduckgo
+        [last modified: 2025-04-22 11:33:08] CACHE-TABLE--radio_browser: radio_browser
+        [last modified: 2025-04-22 11:40:06] CACHE-TABLE--soundcloud: soundcloud
+        [last modified: 2025-04-22 11:40:06] CACHE-TABLE--wolframalpha: wolframalpha
+
+    These properties provide information about the state of the ExpireCache and
+    control the behavior.  For example, the maintenance intervals are controlled by
+    the last modification date of the LAST_MAINTENANCE property and the hash value
+    of the password can be used to detect whether the password has been changed (in
+    this case the DB entries can no longer be decrypted and the entire cache must be
+    discarded).
+    """
+
+    def __init__(self, engine_name: str, expire: int | None = None):
+        self.expire = expire or ENGINES_CACHE.cfg.MAXHOLD_TIME
+        _valid = "-_." + string.ascii_letters + string.digits
+        self.table_name = "".join([c if c in _valid else "_" for c in engine_name])
+
+    def set(self, key: str, value: Any, expire: int | None = None) -> bool:
+        return ENGINES_CACHE.set(
+            key=key,
+            value=value,
+            expire=expire or self.expire,
+            ctx=self.table_name,
+        )
+
+    def get(self, key: str, default=None) -> Any:
+        return ENGINES_CACHE.get(key, default=default, ctx=self.table_name)
+
+    def secret_hash(self, name: str | bytes) -> str:
+        return ENGINES_CACHE.secret_hash(name=name)
+
+
 class Engine:  # pylint: disable=too-few-public-methods
     """Class of engine instances build from YAML settings.
 

searx/enginelib/__main__.py

@@ -0,0 +1,21 @@
+"""Implementation of a command line for development purposes.  To start a
+command, switch to the environment and run library module as a script::
+
+   $ ./manage pyenv.cmd bash --norc --noprofile
+   (py3) python -m searx.enginelib --help
+
+The following commands can be used for maintenance and introspection
+(development) of the engine cache::
+
+   (py3) python -m searx.enginelib cache state
+   (py3) python -m searx.enginelib cache maintenance
+
+"""
+
+import typer
+
+from .. import enginelib
+
+app = typer.Typer()
+app.add_typer(enginelib.app, name="cache", help="Commands related to the cache of the engines.")
+app()

searx/engines/demo_offline.py

@@ -15,6 +15,7 @@ close to the implementation, its just a simple example.  To get in use of this
 import json
 
 from searx.result_types import EngineResults
+from searx.enginelib import EngineCache
 
 engine_type = 'offline'
 categories = ['general']
@@ -32,14 +33,18 @@ about = {
 # if there is a need for globals, use a leading underline
 _my_offline_engine: str = ""
 
+CACHE: EngineCache
+"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
+seconds."""
 
-def init(engine_settings=None):
+
+def init(engine_settings):
     """Initialization of the (offline) engine.  The origin of this demo engine is a
     simple json string which is loaded in this example while the engine is
-    initialized.
+    initialized."""
+    global _my_offline_engine, CACHE  # pylint: disable=global-statement
 
-    """
-    global _my_offline_engine  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])  # type:ignore
 
     _my_offline_engine = (
         '[ {"value": "%s"}'
@@ -57,8 +62,8 @@ def search(query, request_params) -> EngineResults:
     results.
     """
     res = EngineResults()
+    count = CACHE.get("count", 0)
 
-    count = 0
     for row in json.loads(_my_offline_engine):
         count += 1
         kvmap = {
@@ -75,4 +80,7 @@ def search(query, request_params) -> EngineResults:
             )
         )
     res.add(res.types.LegacyResult(number_of_results=count))
+
+    # cache counter value for 20sec
+    CACHE.set("count", count, expire=20)
     return res

searx/engines/duckduckgo.py

@@ -6,16 +6,17 @@ DuckDuckGo WEB
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-import re
-from urllib.parse import quote_plus
 import json
+import re
+import typing
+
+from urllib.parse import quote_plus
+
 import babel
 import lxml.html
 
 from searx import (
     locales,
-    redislib,
     external_bang,
 )
 from searx.utils import (
@@ -25,12 +26,12 @@ from searx.utils import (
     extract_text,
 )
 from searx.network import get  # see https://github.com/searxng/searxng/issues/762
-from searx import redisdb
 from searx.enginelib.traits import EngineTraits
+from searx.enginelib import EngineCache
 from searx.exceptions import SearxEngineCaptchaException
 from searx.result_types import EngineResults
 
-if TYPE_CHECKING:
+if typing.TYPE_CHECKING:
     import logging
 
     logger: logging.Logger
@@ -61,28 +62,18 @@ url = "https://html.duckduckgo.com/html"
 
 time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
 form_data = {'v': 'l', 'api': 'd.js', 'o': 'json'}
-__CACHE = []
+
+CACHE: EngineCache
+"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
+seconds."""
 
 
-def _cache_key(query: str, region: str):
-    return 'SearXNG_ddg_web_vqd' + redislib.secret_hash(f"{query}//{region}")
+def init(_):  # pylint: disable=unused-argument
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache("duckduckgo")  # type:ignore
 
 
-def cache_vqd(query: str, region: str, value: str):
-    """Caches a ``vqd`` value from a query."""
-    c = redisdb.client()
-    if c:
-        logger.debug("VALKEY cache vqd value: %s (%s)", value, region)
-        c.set(_cache_key(query, region), value, ex=600)
-
-    else:
-        logger.debug("MEM cache vqd value: %s (%s)", value, region)
-        if len(__CACHE) > 100:  # cache vqd from last 100 queries
-            __CACHE.pop(0)
-        __CACHE.append((_cache_key(query, region), value))
-
-
-def get_vqd(query: str, region: str, force_request: bool = False):
+def get_vqd(query: str, region: str, force_request: bool = False) -> str:
     """Returns the ``vqd`` that fits to the *query*.
 
     :param query: The query term
@@ -114,31 +105,34 @@ def get_vqd(query: str, region: str, force_request: bool = False):
     seems the block list is a sliding window: to get my IP rid from the bot list
     I had to cool down my IP for 1h (send no requests from that IP to DDG).
     """
-    key = _cache_key(query, region)
-
-    c = redisdb.client()
-    if c:
-        value = c.get(key)
-        if value or value == b'':
-            value = value.decode('utf-8')  # type: ignore
-            logger.debug("re-use CACHED vqd value: %s", value)
+    key = CACHE.secret_hash(f"{query}//{region}")
+    value = CACHE.get(key=key)
+    if value is not None and not force_request:
+        logger.debug("vqd: re-use cached value: %s", value)
         return value
 
-    for k, value in __CACHE:
-        if k == key:
-            logger.debug("MEM re-use CACHED vqd value: %s", value)
-            return value
-
-    if force_request:
+    logger.debug("vqd: request value from from duckduckgo.com")
     resp = get(f'https://duckduckgo.com/?q={quote_plus(query)}')
     if resp.status_code == 200:  # type: ignore
         value = extr(resp.text, 'vqd="', '"')  # type: ignore
         if value:
-                logger.debug("vqd value from DDG request: %s", value)
-                cache_vqd(query, region, value)
+            logger.debug("vqd value from duckduckgo.com request: '%s'", value)
+        else:
+            logger.error("vqd: can't parse value from ddg response (return empty string)")
+            return ""
+    else:
+        logger.error("vqd: got HTTP %s from duckduckgo.com", resp.status_code)
+
+    if value:
+        CACHE.set(key=key, value=value)
+    else:
+        logger.error("vqd value from duckduckgo.com ", resp.status_code)
     return value
 
-    return None
+
+def set_vqd(query: str, region: str, value: str):
+    key = CACHE.secret_hash(f"{query}//{region}")
+    CACHE.set(key=key, value=value, expire=3600)
 
 
 def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
@@ -373,8 +367,11 @@ def response(resp) -> EngineResults:
         # some locales (at least China) does not have a "next page" button
         form = form[0]
         form_vqd = eval_xpath(form, '//input[@name="vqd"]/@value')[0]
-
-        cache_vqd(resp.search_params['data']['q'], resp.search_params['data']['kl'], form_vqd)
+        set_vqd(
+            query=resp.search_params['data']['q'],
+            region=resp.search_params['data']['kl'],
+            value=str(form_vqd),
+        )
 
     # just select "web-result" and ignore results of class "result--ad result--ad--small"
     for div_result in eval_xpath(doc, '//div[@id="links"]/div[contains(@class, "web-result")]'):
@@ -401,7 +398,7 @@ def response(resp) -> EngineResults:
         results.add(
             results.types.Answer(
                 answer=zero_click,
-                url=eval_xpath_getindex(doc, '//div[@id="zero_click_abstract"]/a/@href', 0),
+                url=eval_xpath_getindex(doc, '//div[@id="zero_click_abstract"]/a/@href', 0),  # type: ignore
             )
         )
 

searx/engines/radio_browser.py

@@ -5,7 +5,9 @@
    https://de1.api.radio-browser.info/#Advanced_station_search
 
 """
+from __future__ import annotations
 
+import typing
 import random
 import socket
 from urllib.parse import urlencode
@@ -13,9 +15,15 @@ import babel
 from flask_babel import gettext
 
 from searx.network import get
+from searx.enginelib import EngineCache
 from searx.enginelib.traits import EngineTraits
 from searx.locales import language_tag
 
+if typing.TYPE_CHECKING:
+    import logging
+
+    logger = logging.getLogger()
+
 traits: EngineTraits
 
 about = {
@@ -52,11 +60,24 @@ none filters are applied. Valid filters are:
 
 """
 
-servers = []
+CACHE: EngineCache
+"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
+seconds."""
 
 
 def init(_):
-    # see https://api.radio-browser.info
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache("radio_browser")
+    server_list()
+
+
+def server_list() -> list[str]:
+
+    servers = CACHE.get("servers", [])
+    if servers:
+        return servers
+
+    # hint: can take up to 40sec!
     ips = socket.getaddrinfo("all.api.radio-browser.info", 80, 0, 0, socket.IPPROTO_TCP)
     for ip_tuple in ips:
         _ip: str = ip_tuple[4][0]  # type: ignore
@@ -65,8 +86,22 @@ def init(_):
         if srv not in servers:
             servers.append(srv)
 
+    # update server list once in 24h
+    CACHE.set(key="servers", value=servers, expire=60 * 60 * 24)
+
+    return servers
+
 
 def request(query, params):
+
+    servers = server_list()
+    if not servers:
+        logger.error("Fetched server list is empty!")
+        params["url"] = None
+        return
+
+    server = random.choice(servers)
+
     args = {
         'name': query,
         'order': 'votes',
@@ -87,8 +122,7 @@ def request(query, params):
             if countrycode in traits.custom['countrycodes']:  # type: ignore
                 args['countrycode'] = countrycode
 
-    params['url'] = f"{random.choice(servers)}/json/stations/search?{urlencode(args)}"
-    return params
+    params['url'] = f"{server}/json/stations/search?{urlencode(args)}"
 
 
 def response(resp):
@@ -154,8 +188,9 @@ def fetch_traits(engine_traits: EngineTraits):
 
     babel_reg_list = get_global("territory_languages").keys()
 
-    language_list = get(f'{servers[0]}/json/languages').json()  # type: ignore
-    country_list = get(f'{servers[0]}/json/countries').json()  # type: ignore
+    server = server_list()[0]
+    language_list = get(f'{server}/json/languages').json()  # type: ignore
+    country_list = get(f'{server}/json/countries').json()  # type: ignore
 
     for lang in language_list:
 

searx/engines/soundcloud.py

@@ -1,14 +1,23 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """SoundCloud is a German audio streaming service."""
+from __future__ import annotations
 
 import re
-from urllib.parse import quote_plus, urlencode
+import typing
 import datetime
 
+from urllib.parse import quote_plus, urlencode
+
 from dateutil import parser
 from lxml import html
 
 from searx.network import get as http_get
+from searx.enginelib import EngineCache
+
+if typing.TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
 
 about = {
     "website": "https://soundcloud.com",
@@ -28,7 +37,6 @@ HTML frontend of the common WEB site.
 """
 
 cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U)
-guest_client_id = ""
 results_per_page = 10
 
 soundcloud_facet = "model"
@@ -48,6 +56,10 @@ app_locale_map = {
     "sv": "sv",
 }
 
+CACHE: EngineCache
+"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
+seconds."""
+
 
 def request(query, params):
 
@@ -55,6 +67,12 @@ def request(query, params):
     # - user_id=451561-497874-703312-310156
     # - app_version=1740727428
 
+    guest_client_id = CACHE.get("guest_client_id")
+    if guest_client_id is None:
+        guest_client_id = get_client_id()
+        if guest_client_id:
+            CACHE.set(key="guest_client_id", value=guest_client_id)
+
     args = {
         "q": query,
         "offset": (params['pageno'] - 1) * results_per_page,
@@ -104,12 +122,12 @@ def response(resp):
     return results
 
 
-def init(engine_settings=None):  # pylint: disable=unused-argument
-    global guest_client_id  # pylint: disable=global-statement
-    guest_client_id = get_client_id()
+def init(engine_settings):  # pylint: disable=unused-argument
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])  # type:ignore
 
 
-def get_client_id() -> str:
+def get_client_id() -> str | None:
 
     client_id = ""
     url = "https://soundcloud.com"
@@ -143,4 +161,4 @@ def get_client_id() -> str:
         logger.info("using client_id '%s' for soundclud queries", client_id)
     else:
         logger.warning("missing valid client_id for soundclud queries")
-    return client_id
+    return client_id or None

searx/engines/startpage.py

@@ -84,7 +84,6 @@ from typing import TYPE_CHECKING, Any
 from collections import OrderedDict
 import re
 from unicodedata import normalize, combining
-from time import time
 from datetime import datetime, timedelta
 from json import loads
 
@@ -97,6 +96,7 @@ from searx.network import get  # see https://github.com/searxng/searxng/issues/7
 from searx.exceptions import SearxEngineCaptchaException
 from searx.locales import region_tag
 from searx.enginelib.traits import EngineTraits
+from searx.enginelib import EngineCache
 
 if TYPE_CHECKING:
     import logging
@@ -159,10 +159,21 @@ search_form_xpath = '//form[@id="search"]'
     </form>
 """
 
-# timestamp of the last fetch of 'sc' code
-sc_code_ts = 0
-sc_code = ''
-sc_code_cache_sec = 30
+
+CACHE: EngineCache
+"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
+seconds."""
+
+
+def init(_):
+    global CACHE  # pylint: disable=global-statement
+
+    # hint: all three startpage engines (WEB, Images & News) can/should use the
+    # same sc_code ..
+    CACHE = EngineCache("startpage")  # type:ignore
+
+
+sc_code_cache_sec = 3600
 """Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
 
 
@@ -176,14 +187,10 @@ def get_sc_code(searxng_locale, params):
 
     Startpage's search form generates a new sc-code on each request.  This
     function scrap a new sc-code from Startpage's home page every
-    :py:obj:`sc_code_cache_sec` seconds.
+    :py:obj:`sc_code_cache_sec` seconds."""
 
-    """
-
-    global sc_code_ts, sc_code  # pylint: disable=global-statement
-
-    if sc_code and (time() < (sc_code_ts + sc_code_cache_sec)):
-        logger.debug("get_sc_code: reuse '%s'", sc_code)
+    sc_code = CACHE.get("SC_CODE", "")
+    if sc_code:
         return sc_code
 
     headers = {**params['headers']}
@@ -233,8 +240,9 @@ def get_sc_code(searxng_locale, params):
             message="get_sc_code: [PR-695] query new sc time-stamp failed! (%s)" % resp.url,  # type: ignore
         ) from exc
 
-    sc_code_ts = time()
+    sc_code = str(sc_code)
     logger.debug("get_sc_code: new value is: %s", sc_code)
+    CACHE.set(key="SC_CODE", value=sc_code, expire=sc_code_cache_sec)
     return sc_code
 
 

searx/engines/wolframalpha_api.py

@@ -5,7 +5,7 @@
 
 from urllib.parse import urlencode
 
-from lxml import etree
+import lxml.etree
 
 # about
 about = {
@@ -72,7 +72,7 @@ def replace_pua_chars(text):
 def response(resp):
     results = []
 
-    search_results = etree.XML(resp.content)
+    search_results = lxml.etree.XML(resp.content)
 
     # return empty array if there are no results
     if search_results.xpath(failure_xpath):

searx/engines/wolframalpha_noapi.py

@@ -3,11 +3,13 @@
  Wolfram|Alpha (Science)
 """
 
+from __future__ import annotations
+
 from json import loads
-from time import time
 from urllib.parse import urlencode
 
 from searx.network import get as http_get
+from searx.enginelib import EngineCache
 
 # about
 about = {
@@ -40,41 +42,39 @@ search_url = (
 
 referer_url = url + 'input/?{query}'
 
-token = {'value': '', 'last_updated': None}
-
 # pods to display as image in infobox
 # this pods do return a plaintext, but they look better and are more useful as images
 image_pods = {'VisualRepresentation', 'Illustration', 'Symbol'}
 
 
-# seems, wolframalpha resets its token in every hour
-def obtain_token():
-    update_time = time() - (time() % 3600)
-    try:
-        token_response = http_get('https://www.wolframalpha.com/input/api/v1/code?ts=9999999999999999999', timeout=2.0)
-        token['value'] = loads(token_response.text)['code']
-        token['last_updated'] = update_time
-    except:  # pylint: disable=bare-except
-        pass
+CACHE: EngineCache
+"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
+seconds."""
+
+
+def init(engine_settings):
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])  # type:ignore
+
+
+def obtain_token() -> str:
+    token = CACHE.get(key="token")
+    if token is None:
+        resp = http_get('https://www.wolframalpha.com/input/api/v1/code?ts=9999999999999999999', timeout=2.0)
+        token = resp.json()["code"]
+        # seems, wolframalpha resets its token in every hour
+        CACHE.set(key="code", value=token, expire=3600)
     return token
 
 
-def init(engine_settings=None):  # pylint: disable=unused-argument
-    obtain_token()
-
-
-# do search-request
 def request(query, params):
-    # obtain token if last update was more than an hour
-    if time() - (token['last_updated'] or 0) > 3600:
-        obtain_token()
-    params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value'])
+    token = obtain_token()
+    params['url'] = search_url.format(query=urlencode({'input': query}), token=token)
     params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query}))
 
     return params
 
 
-# get response from search-request
 def response(resp):
     results = []
 

searx/favicons/__init__.py

@@ -1,5 +1,12 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Implementations for providing the favicons in SearXNG"""
+"""Implementations for providing the favicons in SearXNG.
+
+There is a command line for developer purposes and for deeper analysis.  Here is
+an example in which the command line is called in the development environment::
+
+  $ ./manage pyenv.cmd bash --norc --noprofile
+  (py3) python -m searx.favicons --help
+"""
 
 from __future__ import annotations
 

searx/favicons/cache.py

@@ -236,6 +236,12 @@ class FaviconCacheSQLite(sqlitedb.SQLiteAppl, FaviconCache):
     model in the SQLite DB is implemented using the abstract class
     :py:obj:`sqlitedb.SQLiteAppl`.
 
+    For introspection of the DB, jump into developer environment and run command
+    to show cache state::
+
+        $ ./manage pyenv.cmd bash --norc --noprofile
+        (py3) python -m searx.favicons cache state
+
     The following configurations are required / supported:
 
     - :py:obj:`FaviconCacheConfig.db_url`
@@ -357,6 +363,10 @@ CREATE TABLE IF NOT EXISTS blob_map (
             if sha256 != FALLBACK_ICON:
                 conn.execute(self.SQL_INSERT_BLOBS, (sha256, bytes_c, mime, data))
             conn.execute(self.SQL_INSERT_BLOB_MAP, (sha256, resolver, authority))
+        # hint: the with context of the connection object closes the transaction
+        # but not the DB connection.  The connection has to be closed by the
+        # caller of self.connect()!
+        conn.close()
 
         return True
 
@@ -376,7 +386,8 @@ CREATE TABLE IF NOT EXISTS blob_map (
             return
         self.properties.set("LAST_MAINTENANCE", "")  # hint: this (also) sets the m_time of the property!
 
-        # do maintenance tasks
+        # Do maintenance tasks.  This can be take a little more time, to avoid
+        # DB locks, etablish a new DB connecton.
 
         with self.connect() as conn:
 
@@ -407,6 +418,12 @@ CREATE TABLE IF NOT EXISTS blob_map (
                     conn.execute("DELETE FROM blob_map WHERE sha256 IN ('%s')" % "','".join(sha_list))
                     logger.debug("dropped %s blobs with total size of %s bytes", len(sha_list), c)
 
+        # Vacuuming the WALs
+        # https://www.theunterminatedstring.com/sqlite-vacuuming/
+
+        conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+        conn.close()
+
     def _query_val(self, sql, default=None):
         val = self.DB.execute(sql).fetchone()
         if val is not None:

searx/sqlitedb.py

@@ -7,20 +7,83 @@
 :py:obj:`SQLiteProperties`:
   Class to manage properties stored in a database.
 
-----
+Examplarical implementations based on :py:obj:`SQLiteAppl`:
 
+:py:obj:`searx.cache.ExpireCacheSQLite` :
+  Cache that manages key/value pairs in a SQLite DB, in which the key/value
+  pairs are deleted after an "expire" time.  This type of cache is used, for
+  example, for the engines, see :py:obj:`searx.enginelib.EngineCache`.
+
+:py:obj:`searx.favicons.cache.FaviconCacheSQLite` :
+  Favicon cache that manages the favicon BLOBs in a SQLite DB.
+
+----
 """
 from __future__ import annotations
 
-import sys
+import abc
+import datetime
 import re
 import sqlite3
+import sys
 import threading
-import abc
+import uuid
 
 from searx import logger
 
-logger = logger.getChild('sqlitedb')
+logger = logger.getChild("sqlitedb")
+
+THREAD_LOCAL = threading.local()
+
+
+class DBSession:
+    """A *thead-local* DB session"""
+
+    @classmethod
+    def get_connect(cls, app: SQLiteAppl) -> sqlite3.Connection:
+        """Returns a thread local DB connection.  The connection is only
+        established once per thread.
+        """
+        if getattr(THREAD_LOCAL, "DBSession_map", None) is None:
+            THREAD_LOCAL.DBSession_map = {}
+
+        session = THREAD_LOCAL.DBSession_map.get(app.db_url)
+        if session is None:
+            session = cls(app)
+        return session.conn
+
+    def __init__(self, app: SQLiteAppl):
+        self.uuid = uuid.uuid4()
+        self.app = app
+        self._conn = None
+        # self.__del__ will be called, when thread ends
+        if getattr(THREAD_LOCAL, "DBSession_map", None) is None:
+            THREAD_LOCAL.DBSession_map = {}
+        THREAD_LOCAL.DBSession_map[self.app.db_url] = self
+
+    @property
+    def conn(self) -> sqlite3.Connection:
+        msg = f"[{threading.current_thread().ident}] DBSession: " f"{self.app.__class__.__name__}({self.app.db_url})"
+        if self._conn is None:
+            self._conn = self.app.connect()
+            logger.debug("%s --> created new connection", msg)
+        # else:
+        #     logger.debug("%s --> already connected", msg)
+
+        return self._conn
+
+    def __del__(self):
+        try:
+            if self._conn is not None:
+                # HINT: Don't use Python's logging facility in a destructor, it
+                # will produce error reports when python aborts the process or
+                # thread, because at this point objects that the logging module
+                # needs, do not exist anymore.
+                # msg = f"DBSession: close [{self.uuid}] {self.app.__class__.__name__}({self.app.db_url})"
+                # logger.debug(msg)
+                self._conn.close()
+        except Exception:  # pylint: disable=broad-exception-caught
+            pass
 
 
 class SQLiteAppl(abc.ABC):
@@ -51,13 +114,18 @@ class SQLiteAppl(abc.ABC):
     """
 
     SQLITE_JOURNAL_MODE = "WAL"
+    """``SQLiteAppl`` applications are optimzed for WAL_ mode, its not recommend
+    to change the journal mode (see :py:obj:`SQLiteAppl.tear_down`).
+
+    .. _WAL: https://sqlite.org/wal.html
+    """
     SQLITE_CONNECT_ARGS = {
         # "timeout": 5.0,
         # "detect_types": 0,
         "check_same_thread": bool(SQLITE_THREADING_MODE != "serialized"),
         "cached_statements": 0,  # https://github.com/python/cpython/issues/118172
         # "uri": False,
-        "autocommit": False,
+        "isolation_level": None,
     }  # fmt:skip
     """Connection arguments (:py:obj:`sqlite3.connect`)
 
@@ -66,10 +134,6 @@ class SQLiteAppl(abc.ABC):
       ``serialized``.  The check is more of a hindrance in this case because it
       would prevent a DB connector from being used in multiple threads.
 
-    ``autocommit``:
-      Is disabled by default.  Note: autocommit option has been added in Python
-      3.12.
-
     ``cached_statements``:
       Is set to ``0`` by default.  Note: Python 3.12+ fetch result are not
       consistent in multi-threading application and causing an API misuse error.
@@ -89,9 +153,17 @@ class SQLiteAppl(abc.ABC):
 
         self.db_url = db_url
         self.properties = SQLiteProperties(db_url)
-        self.thread_local = threading.local()
         self._init_done = False
         self._compatibility()
+        # atexit.register(self.tear_down)
+
+    # def tear_down(self):
+    #     """:ref:`Vacuuming the WALs` upon normal interpreter termination
+    #     (:py:obj:`atexit.register`).
+
+    #     .. _SQLite: Vacuuming the WALs: https://www.theunterminatedstring.com/sqlite-vacuuming/
+    #     """
+    #     self.DB.execute("PRAGMA wal_checkpoint(TRUNCATE)")
 
     def _compatibility(self):
 
@@ -113,19 +185,31 @@ class SQLiteAppl(abc.ABC):
                 "SQLite runtime library version %s is not supported (require >= 3.35)", sqlite3.sqlite_version
             )
 
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.Connection(self.db_url, **self.SQLITE_CONNECT_ARGS)  # type: ignore
+        conn.execute(f"PRAGMA journal_mode={self.SQLITE_JOURNAL_MODE}")
+        self.register_functions(conn)
+        return conn
+
     def connect(self) -> sqlite3.Connection:
         """Creates a new DB connection (:py:obj:`SQLITE_CONNECT_ARGS`).  If not
-        already done, the DB schema is set up
+        already done, the DB schema is set up.  The caller must take care of
+        closing the resource.  Alternatively, :py:obj:`SQLiteAppl.DB` can also
+        be used (the resource behind `self.DB` is automatically closed when the
+        process or thread is terminated).
         """
         if sys.version_info < (3, 12):
             # Prior Python 3.12 there is no "autocommit" option
             self.SQLITE_CONNECT_ARGS.pop("autocommit", None)
 
-        self.init()
-        logger.debug("%s: connect to DB: %s // %s", self.__class__.__name__, self.db_url, self.SQLITE_CONNECT_ARGS)
-        conn = sqlite3.Connection(self.db_url, **self.SQLITE_CONNECT_ARGS)  # type: ignore
-        conn.execute(f"PRAGMA journal_mode={self.SQLITE_JOURNAL_MODE}")
-        self.register_functions(conn)
+        msg = (
+            f"[{threading.current_thread().ident}] {self.__class__.__name__}({self.db_url})"
+            f" {self.SQLITE_CONNECT_ARGS} // {self.SQLITE_JOURNAL_MODE}"
+        )
+        logger.debug(msg)
+
+        with self._connect() as conn:
+            self.init(conn)
         return conn
 
     def register_functions(self, conn):
@@ -150,7 +234,7 @@ class SQLiteAppl(abc.ABC):
         .. _re.search: https://docs.python.org/3/library/re.html#re.search
         """
 
-        conn.create_function('regexp', 2, lambda x, y: 1 if re.search(x, y) else 0, deterministic=True)
+        conn.create_function("regexp", 2, lambda x, y: 1 if re.search(x, y) else 0, deterministic=True)
 
     @property
     def DB(self) -> sqlite3.Connection:
@@ -168,57 +252,66 @@ class SQLiteAppl(abc.ABC):
             https://docs.python.org/3/library/sqlite3.html#sqlite3-controlling-transactions
         """
 
-        if getattr(self.thread_local, 'DB', None) is None:
-            self.thread_local.DB = self.connect()
+        conn = None
 
-        # Theoretically it is possible to reuse the DB cursor across threads as
-        # of Python 3.12, in practice the threading of the cursor seems to me to
-        # be so faulty that I prefer to establish one connection per thread
-
-        self.thread_local.DB.commit()
-        return self.thread_local.DB
-
-        # In "serialized" mode, SQLite can be safely used by multiple threads
-        # with no restriction.
-        #
-        # if self.SQLITE_THREADING_MODE != "serialized":
-        #     if getattr(self.thread_local, 'DB', None) is None:
-        #         self.thread_local.DB = self.connect()
-        #     return self.thread_local.DB
+        if self.SQLITE_THREADING_MODE == "serialized":
+            # Theoretically it is possible to reuse the DB cursor across threads
+            # as of Python 3.12, in practice the threading of the cursor seems
+            # to me a little faulty that I prefer to establish one connection
+            # per thread.
             #
+            # may we can activate this code one day ..
             # if self._DB is None:
-        #     self._DB = self.connect()  # pylint: disable=attribute-defined-outside-init
-        # return self._DB
+            #     self._DB = self.connect()
+            # conn = self._DB
+            conn = DBSession.get_connect(self)
+        else:
+            conn = DBSession.get_connect(self)
 
-    def init(self):
+        # Since more than one instance of SQLiteAppl share the same DB
+        # connection, we need to make sure that each SQLiteAppl instance has run
+        # its init method at least once.
+        self.init(conn)
+
+        return conn
+
+    def init(self, conn: sqlite3.Connection) -> bool:
         """Initializes the DB schema and properties, is only executed once even
-        if called several times."""
+        if called several times.
+
+        If the initialization has not yet taken place, it is carried out and a
+        `True` is returned to the caller at the end.  If the initialization has
+        already been carried out in the past, `False` is returned.
+        """
 
         if self._init_done:
-            return
+            return False
         self._init_done = True
 
         logger.debug("init DB: %s", self.db_url)
-        self.properties.init()
+        self.properties.init(conn)
+
         ver = self.properties("DB_SCHEMA")
         if ver is None:
-            with self.properties.DB:
-                self.create_schema(self.properties.DB)
+            with conn:
+                self.create_schema(conn)
         else:
             ver = int(ver)
             if ver != self.DB_SCHEMA:
                 raise sqlite3.DatabaseError("Expected DB schema v%s, DB schema is v%s" % (self.DB_SCHEMA, ver))
             logger.debug("DB_SCHEMA = %s", ver)
 
-    def create_schema(self, conn):
+        return True
+
+    def create_schema(self, conn: sqlite3.Connection):
 
         logger.debug("create schema ..")
+        self.properties.set("DB_SCHEMA", self.DB_SCHEMA)
+        self.properties.set("LAST_MAINTENANCE", "")
         with conn:
             for table_name, sql in self.DDL_CREATE_TABLES.items():
                 conn.execute(sql)
                 self.properties.set(f"Table {table_name} created", table_name)
-                self.properties.set("DB_SCHEMA", self.DB_SCHEMA)
-                self.properties.set("LAST_MAINTENANCE", "")
 
 
 class SQLiteProperties(SQLiteAppl):
@@ -253,33 +346,32 @@ CREATE TABLE IF NOT EXISTS properties (
         "    ON CONFLICT(name) DO UPDATE"
         "   SET value=excluded.value, m_time=strftime('%s', 'now')"
     )
+    SQL_DELETE = "DELETE FROM properties WHERE name = ?"
     SQL_TABLE_EXISTS = (
         "SELECT name FROM sqlite_master"
         " WHERE type='table' AND name='properties'"
     )  # fmt:skip
     SQLITE_CONNECT_ARGS = dict(SQLiteAppl.SQLITE_CONNECT_ARGS)
-    SQLITE_CONNECT_ARGS["autocommit"] = True  # This option has no effect before Python 3.12
 
     def __init__(self, db_url: str):  # pylint: disable=super-init-not-called
 
         self.db_url = db_url
-        self.thread_local = threading.local()
         self._init_done = False
         self._compatibility()
 
-    def init(self):
+    def init(self, conn: sqlite3.Connection) -> bool:
         """Initializes DB schema of the properties in the DB."""
 
         if self._init_done:
-            return
+            return False
         self._init_done = True
         logger.debug("init properties of DB: %s", self.db_url)
-        with self.DB as conn:
         res = conn.execute(self.SQL_TABLE_EXISTS)
         if res.fetchone() is None:  # DB schema needs to be be created
             self.create_schema(conn)
+        return True
 
-    def __call__(self, name, default=None):
+    def __call__(self, name: str, default=None):
         """Returns the value of the property ``name`` or ``default`` if property
         not exists in DB."""
 
@@ -288,36 +380,47 @@ CREATE TABLE IF NOT EXISTS properties (
             return default
         return res[0]
 
-    def set(self, name, value):
+    def set(self, name: str, value: str | int):
         """Set ``value`` of property ``name`` in DB.  If property already
         exists, update the ``m_time`` (and the value)."""
 
+        with self.DB:
             self.DB.execute(self.SQL_SET, (name, value))
 
-        if sys.version_info <= (3, 12):
-            # Prior Python 3.12 there is no "autocommit" option / lets commit
-            # explicitely.
-            self.DB.commit()
+    def delete(self, name: str) -> int:
+        """Delete of property ``name`` from DB."""
+        with self.DB:
+            cur = self.DB.execute(self.SQL_DELETE, (name,))
+        return cur.rowcount
 
-    def row(self, name, default=None):
+    def row(self, name: str, default=None):
         """Returns the DB row of property ``name`` or ``default`` if property
         not exists in DB."""
 
-        cur = self.DB.cursor()
-        cur.execute("SELECT * FROM properties WHERE name = ?", (name,))
-        res = cur.fetchone()
-        if res is None:
+        res = self.DB.execute("SELECT * FROM properties WHERE name = ?", (name,))
+        row = res.fetchone()
+        if row is None:
             return default
-        col_names = [column[0] for column in cur.description]
-        return dict(zip(col_names, res))
 
-    def m_time(self, name, default: int = 0) -> int:
+        col_names = [column[0] for column in row.description]
+        return dict(zip(col_names, row))
+
+    def m_time(self, name: str, default: int = 0) -> int:
         """Last modification time of this property."""
-        res = self.DB.execute(self.SQL_M_TIME, (name,)).fetchone()
-        if res is None:
+        res = self.DB.execute(self.SQL_M_TIME, (name,))
+        row = res.fetchone()
+        if row is None:
             return default
-        return int(res[0])
+        return int(row[0])
 
     def create_schema(self, conn):
         with conn:
             conn.execute(self.DDL_PROPERTIES)
+
+    def __str__(self) -> str:
+        lines = []
+        for row in self.DB.execute("SELECT name, value, m_time FROM properties"):
+            name, value, m_time = row
+            m_time = datetime.datetime.fromtimestamp(m_time).strftime("%Y-%m-%d %H:%M:%S")
+            lines.append(f"[last modified: {m_time}] {name:20s}: {value}")
+        return "\n".join(lines)