[mod] migrate from Redis to Valkey (#4795)

This patch migrates from `redis==5.2.1` [1] to `valkey==6.1.0` [2].

The migration to valkey is necessary because the company behind Redis has decided
to abandon the open source license. After experiencing a drop in user numbers,
they now want to run it under a dual license again. But this move demonstrates
once again how unreliable the company is and how it treats open source
developers.

To review first, read the docs::

    $ make docs.live

Follow the instructions to remove redis:

- http://0.0.0.0:8000/admin/settings/settings_redis.html

Config and install a local valkey DB:

- http://0.0.0.0:8000/admin/settings/settings_valkey.html

[1] https://pypi.org/project/redis/
[2] https://pypi.org/project/valkey/

Co-authored-by: HLFH <gaspard@dhautefeuille.eu>
Co-authored-by: Markus Heiser <markus.heiser@darmarit.de>

searx/search/checker/background.py

@@ -8,18 +8,18 @@ import os
 import signal
 from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
 
-import redis.exceptions
+import valkey.exceptions
 
 from searx import logger, settings, sxng_debug
-from searx.redisdb import client as get_redis_client
+from searx.valkeydb import client as get_valkey_client
 from searx.exceptions import SearxSettingsException
 from searx.search.processors import PROCESSORS
 from searx.search.checker import Checker
 from searx.search.checker.scheduler import scheduler_function
 
 
-REDIS_RESULT_KEY = 'SearXNG_checker_result'
-REDIS_LOCK_KEY = 'SearXNG_checker_lock'
+VALKEY_RESULT_KEY = 'SearXNG_checker_result'
+VALKEY_LOCK_KEY = 'SearXNG_checker_lock'
 
 
 CheckerResult = Union['CheckerOk', 'CheckerErr', 'CheckerOther']
@@ -77,23 +77,23 @@ def _get_interval(every: Any, error_msg: str) -> Tuple[int, int]:
 
 
 def get_result() -> CheckerResult:
-    client = get_redis_client()
+    client = get_valkey_client()
     if client is None:
-        # without Redis, the checker is disabled
+        # without Valkey, the checker is disabled
         return {'status': 'disabled'}
-    serialized_result: Optional[bytes] = client.get(REDIS_RESULT_KEY)
+    serialized_result: Optional[bytes] = client.get(VALKEY_RESULT_KEY)
     if serialized_result is None:
-        # the Redis key does not exist
+        # the Valkey key does not exist
         return {'status': 'unknown'}
     return json.loads(serialized_result)
 
 
 def _set_result(result: CheckerResult):
-    client = get_redis_client()
+    client = get_valkey_client()
     if client is None:
-        # without Redis, the function does nothing
+        # without Valkey, the function does nothing
         return
-    client.set(REDIS_RESULT_KEY, json.dumps(result))
+    client.set(VALKEY_RESULT_KEY, json.dumps(result))
 
 
 def _timestamp():
@@ -102,9 +102,9 @@ def _timestamp():
 
 def run():
     try:
-        # use a Redis lock to make sure there is no checker running at the same time
+        # use a Valkey lock to make sure there is no checker running at the same time
         # (this should not happen, this is a safety measure)
-        with get_redis_client().lock(REDIS_LOCK_KEY, blocking_timeout=60, timeout=3600):
+        with get_valkey_client().lock(VALKEY_LOCK_KEY, blocking_timeout=60, timeout=3600):
             logger.info('Starting checker')
             result: CheckerOk = {'status': 'ok', 'engines': {}, 'timestamp': _timestamp()}
             for name, processor in PROCESSORS.items():
@@ -118,7 +118,7 @@ def run():
 
             _set_result(result)
             logger.info('Check done')
-    except redis.exceptions.LockError:
+    except valkey.exceptions.LockError:
         _set_result({'status': 'error', 'timestamp': _timestamp()})
         logger.exception('Error while running the checker')
     except Exception:  # pylint: disable=broad-except
@@ -149,9 +149,9 @@ def initialize():
         logger.info('Checker scheduler is disabled')
         return
 
-    # make sure there is a Redis connection
-    if get_redis_client() is None:
-        logger.error('The checker requires Redis')
+    # make sure there is a Valkey connection
+    if get_valkey_client() is None:
+        logger.error('The checker requires Valkey')
         return
 
     # start the background scheduler

searx/search/checker/scheduler.lua

@@ -2,9 +2,9 @@
 --
 -- This script is not a string in scheduler.py, so editors can provide syntax highlighting.
 
--- The Redis KEY is defined here and not in Python on purpose:
+-- The Valkey KEY is defined here and not in Python on purpose:
 -- only this LUA script can read and update this key to avoid lock and concurrency issues.
-local redis_key = 'SearXNG_checker_next_call_ts'
+local valkey_key = 'SearXNG_checker_next_call_ts'
 
 local now = redis.call('TIME')[1]
 local start_after_from = ARGV[1]
@@ -12,14 +12,14 @@ local start_after_to = ARGV[2]
 local every_from = ARGV[3]
 local every_to = ARGV[4]
 
-local next_call_ts = redis.call('GET', redis_key)
+local next_call_ts = redis.call('GET', valkey_key)
 
 if (next_call_ts == false or next_call_ts == nil) then
-    -- the scheduler has never run on this Redis instance, so:
+    -- the scheduler has never run on this Valkey instance, so:
     -- 1/ the scheduler does not run now
     -- 2/ the next call is a random time between start_after_from and start_after_to
     local initial_delay = math.random(start_after_from, start_after_to)
-    redis.call('SET', redis_key, now + initial_delay)
+    redis.call('SET', valkey_key, now + initial_delay)
     return { false, initial_delay }
 end
 
@@ -31,6 +31,6 @@ if call_now then
     -- the checker runs now, define the timestamp of the next call:
     -- this is a random delay between every_from and every_to
     local periodic_delay = math.random(every_from, every_to)
-    next_call_ts = redis.call('INCRBY', redis_key, periodic_delay)
+    next_call_ts = redis.call('INCRBY', valkey_key, periodic_delay)
 end
 return { call_now, next_call_ts - now }

searx/search/checker/scheduler.py

@@ -1,11 +1,11 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # pylint: disable=missing-module-docstring
-"""Lame scheduler which use Redis as a source of truth:
-* the Redis key SearXNG_checker_next_call_ts contains the next time the embedded checker should run.
-* to avoid lock, a unique Redis script reads and updates the Redis key SearXNG_checker_next_call_ts.
-* this Redis script returns a list of two elements:
+"""Lame scheduler which use Valkey as a source of truth:
+* the Valkey key SearXNG_checker_next_call_ts contains the next time the embedded checker should run.
+* to avoid lock, a unique Valkey script reads and updates the Valkey key SearXNG_checker_next_call_ts.
+* this Valkey script returns a list of two elements:
    * the first one is a boolean. If True, the embedded checker must run now in this worker.
-   * the second element is the delay in second to wait before the next call to the Redis script.
+   * the second element is the delay in second to wait before the next call to the Valkey script.
 
 This scheduler is not generic on purpose: if more feature are required, a dedicate scheduler must be used
 (= a better scheduler should not use the web workers)
@@ -16,8 +16,8 @@ import time
 from pathlib import Path
 from typing import Callable
 
-from searx.redisdb import client as get_redis_client
-from searx.redislib import lua_script_storage
+from searx.valkeydb import client as get_valkey_client
+from searx.valkeylib import lua_script_storage
 
 
 logger = logging.getLogger('searx.search.checker')
@@ -29,7 +29,7 @@ def scheduler_function(start_after_from: int, start_after_to: int, every_from: i
     """Run the checker periodically. The function never returns.
 
     Parameters:
-    * start_after_from and start_after_to: when to call "callback" for the first on the Redis instance
+    * start_after_from and start_after_to: when to call "callback" for the first on the Valkey instance
     * every_from and every_to: after the first call, how often to call "callback"
 
     There is no issue:
@@ -38,11 +38,11 @@ def scheduler_function(start_after_from: int, start_after_to: int, every_from: i
     """
     scheduler_now_script = SCHEDULER_LUA.open().read()
     while True:
-        # ask the Redis script what to do
+        # ask the Valkey script what to do
         # the script says
         # * if the checker must run now.
         # * how to long to way before calling the script again (it can be call earlier, but not later).
-        script = lua_script_storage(get_redis_client(), scheduler_now_script)
+        script = lua_script_storage(get_valkey_client(), scheduler_now_script)
         call_now, wait_time = script(args=[start_after_from, start_after_to, every_from, every_to])
 
         # does the worker run the checker now?