[mod] upgrade to httpx 0.28.1 (#4674)

The global variable CACHE is not initialized when DDG images or DDG videos
import the get_vqd() function (please remember: the engine modules are imported
using the importlib method and not via the `import` keyword).

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

Makefile

@@ -54,7 +54,7 @@ ci.test: test.yamllint test.black test.types.ci  test.pylint test.unit test.robo
 test:    test.yamllint test.black test.types.dev test.pylint test.unit test.robot test.rst test.shell
 test.shell:
 	$(Q)shellcheck -x -s dash \
-		container/docker-entrypoint.sh
+		container/entrypoint.sh
 	$(Q)shellcheck -x -s bash \
 		utils/brand.sh \
 		$(MTOOLS) \

container/Dockerfile

@@ -10,11 +10,9 @@ RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \
 COPY ./searx/ ./searx/
 
 ARG TIMESTAMP_SETTINGS="0"
-ARG TIMESTAMP_UWSGI="0"
 
 RUN python -m compileall -q searx \
  && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \
- && touch -c --date=@$TIMESTAMP_UWSGI ./container/uwsgi.ini \
  && find ./searx/static \
     \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" -o -name "*.ttf" -o -name "*.eot" \) \
     -type f -exec gzip -9 -k {} + -exec brotli --best {} +
@@ -29,7 +27,12 @@ ARG LABEL_VCS_URL="unspecified"
 
 COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/
 COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/
-COPY --chown=searxng:searxng ./container/ ./container/
+COPY --chown=searxng:searxng ./container/config/ ./.template/
+COPY --chown=searxng:searxng ./container/entrypoint.sh ./entrypoint.sh
+
+ARG TIMESTAMP_UWSGI="0"
+
+RUN touch -c --date=@$TIMESTAMP_UWSGI ./.template/uwsgi.ini
 
 LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \
       org.opencontainers.image.created="$LABEL_DATE" \
@@ -59,4 +62,4 @@ EXPOSE 8080
 
 HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1
 
-ENTRYPOINT ["/usr/local/searxng/container/docker-entrypoint.sh"]
+ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]

container/docker-entrypoint.sh

@@ -1,127 +0,0 @@
-#!/bin/sh
-
-help() {
-    cat <<EOF
-Command line:
-  -h  Display this help
-  -d  Dry run to update the configuration files.
-  -f  Always update on the configuration files (existing files are renamed with
-      the .old suffix).  Without this option, the new configuration files are
-      copied with the .new suffix
-Environment variables:
-  INSTANCE_NAME settings.yml : general.instance_name
-  AUTOCOMPLETE  settings.yml : search.autocomplete
-  BASE_URL      settings.yml : server.base_url
-
-Volume:
-  /etc/searxng  the docker entry point copies settings.yml and uwsgi.ini in
-                this directory (see the -f command line option)"
-
-EOF
-}
-
-# Parse command line
-FORCE_CONF_UPDATE=0
-DRY_RUN=0
-
-while getopts "fdh" option
-do
-    case $option in
-
-        f) FORCE_CONF_UPDATE=1 ;;
-        d) DRY_RUN=1 ;;
-
-        h)
-            help
-            exit 0
-            ;;
-        *)
-            echo "unknow option ${option}"
-            exit 42
-            ;;
-    esac
-done
-
-echo "SearXNG version $SEARXNG_VERSION"
-
-# helpers to update the configuration files
-patch_uwsgi_settings() {
-    CONF="$1"
-
-    # update uwsg.ini
-    sed -i \
-        -e "s|workers = .*|workers = ${UWSGI_WORKERS:-%k}|g" \
-        -e "s|threads = .*|threads = ${UWSGI_THREADS:-4}|g" \
-        "${CONF}"
-}
-
-patch_searxng_settings() {
-    CONF="$1"
-
-    # Make sure that there is trailing slash at the end of BASE_URL
-    # see https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion
-    export BASE_URL="${BASE_URL%/}/"
-
-    # update settings.yml
-    sed -i \
-        -e "s|base_url: false|base_url: ${BASE_URL}|g" \
-        -e "s/instance_name: \"SearXNG\"/instance_name: \"${INSTANCE_NAME}\"/g" \
-        -e "s/autocomplete: \"\"/autocomplete: \"${AUTOCOMPLETE}\"/g" \
-        -e "s/ultrasecretkey/$(head -c 24 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9')/g" \
-        "${CONF}"
-}
-
-# FIXME: Always use "searxng:searxng" ownership
-update_conf() {
-    FORCE_CONF_UPDATE=$1
-    CONF="$2"
-    NEW_CONF="${2}.new"
-    OLD_CONF="${2}.old"
-    REF_CONF="$3"
-    PATCH_REF_CONF="$4"
-
-    if [ -f "${CONF}" ]; then
-        if [ "${REF_CONF}" -nt "${CONF}" ]; then
-            # There is a new version
-            if [ "$FORCE_CONF_UPDATE" -ne 0 ]; then
-                # Replace the current configuration
-                printf '⚠️  Automatically update %s to the new version\n' "${CONF}"
-                if [ ! -f "${OLD_CONF}" ]; then
-                    printf 'The previous configuration is saved to %s\n' "${OLD_CONF}"
-                    mv "${CONF}" "${OLD_CONF}"
-                fi
-                cp "${REF_CONF}" "${CONF}"
-                $PATCH_REF_CONF "${CONF}"
-            else
-                # Keep the current configuration
-                printf '⚠️  Check new version %s to make sure SearXNG is working properly\n' "${NEW_CONF}"
-                cp "${REF_CONF}" "${NEW_CONF}"
-                $PATCH_REF_CONF "${NEW_CONF}"
-            fi
-        else
-            printf 'Use existing %s\n' "${CONF}"
-        fi
-    else
-        printf 'Create %s\n' "${CONF}"
-        cp "${REF_CONF}" "${CONF}"
-        $PATCH_REF_CONF "${CONF}"
-    fi
-}
-
-# make sure there are uwsgi settings
-update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searxng/container/uwsgi.ini" "patch_uwsgi_settings"
-
-# make sure there are searxng settings
-update_conf "${FORCE_CONF_UPDATE}" "${SEARXNG_SETTINGS_PATH}" "/usr/local/searxng/searx/settings.yml" "patch_searxng_settings"
-
-# dry run (to update configuration files, then inspect them)
-if [ $DRY_RUN -eq 1 ]; then
-    printf 'Dry run\n'
-    exit
-fi
-
-printf 'Listen on %s\n' "${BIND_ADDRESS}"
-
-# Start uwsgi
-# TODO: "--http-socket" will be removed in the future (see uwsgi.ini.new config file): https://github.com/searxng/searxng/pull/4578
-exec /usr/local/searxng/venv/bin/uwsgi --http-socket "${BIND_ADDRESS}" "${UWSGI_SETTINGS_PATH}"

container/entrypoint.sh

@@ -0,0 +1,166 @@
+#!/bin/sh
+# shellcheck shell=dash
+set -eu
+
+check_file() {
+    local target="$1"
+
+    if [ ! -f "$target" ]; then
+        cat <<EOF
+!!!
+!!! ERROR
+!!! "$target" is not a valid file, exiting...
+!!!
+EOF
+        exit 127
+    fi
+}
+
+check_directory() {
+    local target="$1"
+
+    if [ ! -d "$target" ]; then
+        cat <<EOF
+!!!
+!!! ERROR
+!!! "$target" is not a valid directory, exiting...
+!!!
+EOF
+        exit 127
+    fi
+}
+
+setup_ownership() {
+    local target="$1"
+    local type="$2"
+
+    case "$type" in
+    file | directory) ;;
+    *)
+        cat <<EOF
+!!!
+!!! ERROR
+!!! "$type" is not a valid type, exiting...
+!!!
+EOF
+        exit 1
+        ;;
+    esac
+
+    if [ "$(stat -c %U:%G "$target")" != "searxng:searxng" ]; then
+        if [ "$(id -u)" -eq 0 ]; then
+            chown -R searxng:searxng "$target"
+        else
+            cat <<EOF
+!!!
+!!! WARNING
+!!! "$target" $type is not owned by "searxng"
+!!! This may cause issues when running SearXNG
+!!!
+!!! Run the container as root to fix this issue automatically
+!!! Alternatively, you can chown the $type manually:
+!!! $ chown -R searxng:searxng "$target"
+!!!
+EOF
+        fi
+    fi
+}
+
+# Apply envs to uwsgi.ini
+setup_uwsgi() {
+    local timestamp
+
+    timestamp=$(stat -c %Y "$UWSGI_SETTINGS_PATH")
+
+    sed -i \
+        -e "s|workers = .*|workers = ${UWSGI_WORKERS:-%k}|g" \
+        -e "s|threads = .*|threads = ${UWSGI_THREADS:-4}|g" \
+        "$UWSGI_SETTINGS_PATH"
+
+    # Restore timestamp
+    touch -c -d "@$timestamp" "$UWSGI_SETTINGS_PATH"
+}
+
+# Apply envs to settings.yml
+setup_searxng() {
+    local timestamp
+
+    timestamp=$(stat -c %Y "$SEARXNG_SETTINGS_PATH")
+
+    # Ensure trailing slash in BASE_URL
+    # https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion
+    export BASE_URL="${BASE_URL%/}/"
+
+    sed -i \
+        -e "s|base_url: false|base_url: ${BASE_URL:-false}|g" \
+        -e "s/instance_name: \"SearXNG\"/instance_name: \"${INSTANCE_NAME:-SearXNG}\"/g" \
+        -e "s/autocomplete: \"\"/autocomplete: \"${AUTOCOMPLETE:-}\"/g" \
+        -e "s/ultrasecretkey/$(head -c 24 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9')/g" \
+        "$SEARXNG_SETTINGS_PATH"
+
+    # Restore timestamp
+    touch -c -d "@$timestamp" "$SEARXNG_SETTINGS_PATH"
+}
+
+# Handle volume mounts
+volume_handler() {
+    local target="$1"
+
+    # Check if it's a valid directory
+    check_directory "$target"
+    setup_ownership "$target" "directory"
+}
+
+# Handle configuration file updates
+config_handler() {
+    local target="$1"
+    local template="$2"
+    local new_template_target="$target.new"
+
+    # Create/Update the configuration file
+    if [ -f "$target" ]; then
+        setup_ownership "$target" "file"
+
+        if [ "$template" -nt "$target" ]; then
+            cp -pfT "$template" "$new_template_target"
+
+            cat <<EOF
+...
+... INFORMATION
+... Update available for "$target"
+... It is recommended to update the configuration file to ensure proper functionality
+...
+... New version placed at "$new_template_target"
+... Please review and merge changes
+...
+EOF
+        fi
+    else
+        cat <<EOF
+...
+... INFORMATION
+... "$target" does not exist, creating from template...
+...
+EOF
+        cp -pfT "$template" "$target"
+    fi
+
+    # Check if it's a valid file
+    check_file "$target"
+}
+
+echo "SearXNG $SEARXNG_VERSION"
+
+# Check for volume mounts
+volume_handler "$CONFIG_PATH"
+volume_handler "$DATA_PATH"
+
+# Check for updates in files
+config_handler "$UWSGI_SETTINGS_PATH" "/usr/local/searxng/.template/uwsgi.ini"
+config_handler "$SEARXNG_SETTINGS_PATH" "/usr/local/searxng/searx/settings.yml"
+
+# Update files
+setup_uwsgi
+setup_searxng
+
+exec /usr/local/searxng/venv/bin/uwsgi --http-socket "$BIND_ADDRESS" "$UWSGI_SETTINGS_PATH"

container/legacy/Dockerfile

@@ -1,5 +1,3 @@
-# For armv7 architecture
-
 FROM docker.io/library/python:3.13-slim AS builder
 
 RUN apt-get update \
@@ -26,11 +24,9 @@ RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \
 COPY ./searx/ ./searx/
 
 ARG TIMESTAMP_SETTINGS=0
-ARG TIMESTAMP_UWSGI=0
 
 RUN python -m compileall -q searx \
  && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \
- && touch -c --date=@$TIMESTAMP_UWSGI ./container/uwsgi.ini \
  && find /usr/local/searxng/searx/static \
     \( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \
     -type f -exec gzip -9 -k {} + -exec brotli --best {} +
@@ -70,7 +66,12 @@ WORKDIR /usr/local/searxng/
 
 COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/
 COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/
-COPY --chown=searxng:searxng ./container/ ./container/
+COPY --chown=searxng:searxng ./container/config/ ./.template/
+COPY --chown=searxng:searxng ./container/entrypoint.sh ./entrypoint.sh
+
+ARG TIMESTAMP_UWSGI="0"
+
+RUN touch -c --date=@$TIMESTAMP_UWSGI ./.template/uwsgi.ini
 
 LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \
       org.opencontainers.image.created=$LABEL_DATE \
@@ -103,4 +104,4 @@ EXPOSE 8080
 
 HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1
 
-ENTRYPOINT ["/usr/local/searxng/container/docker-entrypoint.sh"]
+ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]

docs/admin/installation-docker.rst

@@ -181,10 +181,10 @@ Command line
    <https://docs.docker.com/engine/reference/run/#foreground>`__.
 
 In the :origin:`Dockerfile` the ENTRYPOINT_ is defined as
-:origin:`container/docker-entrypoint.sh`
+:origin:`container/entrypoint.sh`
 
 .. code:: sh
 
     docker run --rm -it searxng/searxng -h
 
-.. program-output:: ../container/docker-entrypoint.sh -h
+.. program-output:: ../container/entrypoint.sh -h

requirements.txt

@@ -7,10 +7,10 @@ lxml==5.4.0
 pygments==2.19.1
 python-dateutil==2.9.0.post0
 pyyaml==6.0.2
-httpx[http2]==0.24.1
+httpx[http2]==0.28.1
+httpx-socks[asyncio]==0.10.0
 Brotli==1.1.0
 uvloop==0.21.0
-httpx-socks[asyncio]==0.7.7
 setproctitle==1.3.6
 redis==5.2.1
 markdown-it-py==3.0.0

searx/engines/duckduckgo.py

@@ -63,14 +63,16 @@ url = "https://html.duckduckgo.com/html"
 time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
 form_data = {'v': 'l', 'api': 'd.js', 'o': 'json'}
 
-CACHE: EngineCache
+_CACHE: EngineCache = None  # type: ignore
 """Persistent (SQLite) key/value cache that deletes its values after ``expire``
 seconds."""
 
 
-def init(_):  # pylint: disable=unused-argument
-    global CACHE  # pylint: disable=global-statement
-    CACHE = EngineCache("duckduckgo")  # type:ignore
+def get_cache():
+    global _CACHE  # pylint: disable=global-statement
+    if _CACHE is None:
+        _CACHE = EngineCache("duckduckgo")  # type:ignore
+    return _CACHE
 
 
 def get_vqd(query: str, region: str, force_request: bool = False) -> str:
@@ -105,8 +107,9 @@ def get_vqd(query: str, region: str, force_request: bool = False) -> str:
     seems the block list is a sliding window: to get my IP rid from the bot list
     I had to cool down my IP for 1h (send no requests from that IP to DDG).
     """
-    key = CACHE.secret_hash(f"{query}//{region}")
-    value = CACHE.get(key=key)
+    cache = get_cache()
+    key = cache.secret_hash(f"{query}//{region}")
+    value = cache.get(key=key)
     if value is not None and not force_request:
         logger.debug("vqd: re-use cached value: %s", value)
         return value
@@ -124,15 +127,16 @@ def get_vqd(query: str, region: str, force_request: bool = False) -> str:
         logger.error("vqd: got HTTP %s from duckduckgo.com", resp.status_code)
 
     if value:
-        CACHE.set(key=key, value=value)
+        cache.set(key=key, value=value)
     else:
         logger.error("vqd value from duckduckgo.com ", resp.status_code)
     return value
 
 
 def set_vqd(query: str, region: str, value: str):
-    key = CACHE.secret_hash(f"{query}//{region}")
-    CACHE.set(key=key, value=value, expire=3600)
+    cache = get_cache()
+    key = cache.secret_hash(f"{query}//{region}")
+    cache.set(key=key, value=value, expire=3600)
 
 
 def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):

searx/network/client.py

@@ -24,7 +24,7 @@ LOOP = None
 SSLCONTEXTS: Dict[Any, SSLContext] = {}
 
 
-def shuffle_ciphers(ssl_context):
+def shuffle_ciphers(ssl_context: SSLContext):
     """Shuffle httpx's default ciphers of a SSL context randomly.
 
     From `What Is TLS Fingerprint and How to Bypass It`_
@@ -41,16 +41,16 @@ def shuffle_ciphers(ssl_context):
        https://www.zenrows.com/blog/what-is-tls-fingerprint#how-to-bypass-tls-fingerprinting
 
     """
-    c_list = httpx._config.DEFAULT_CIPHERS.split(':')  # pylint: disable=protected-access
+    c_list = [cipher["name"] for cipher in ssl_context.get_ciphers()]
     sc_list, c_list = c_list[:3], c_list[3:]
     random.shuffle(c_list)
     ssl_context.set_ciphers(":".join(sc_list + c_list))
 
 
-def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http2=False):
-    key = (proxy_url, cert, verify, trust_env, http2)
+def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True):
+    key = (proxy_url, cert, verify, trust_env)
     if key not in SSLCONTEXTS:
-        SSLCONTEXTS[key] = httpx.create_ssl_context(cert, verify, trust_env, http2)
+        SSLCONTEXTS[key] = httpx.create_ssl_context(verify, cert, trust_env)
     shuffle_ciphers(SSLCONTEXTS[key])
     return SSLCONTEXTS[key]
 
@@ -120,7 +120,7 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit
         rdns = True
 
     proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)
-    verify = get_sslcontexts(proxy_url, None, verify, True, http2) if verify is True else verify
+    verify = get_sslcontexts(proxy_url, None, verify, True) if verify is True else verify
     return AsyncProxyTransportFixed(
         proxy_type=proxy_type,
         proxy_host=proxy_host,
@@ -138,7 +138,7 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit
 
 
 def get_transport(verify, http2, local_address, proxy_url, limit, retries):
-    verify = get_sslcontexts(None, None, verify, True, http2) if verify is True else verify
+    verify = get_sslcontexts(None, None, verify, True) if verify is True else verify
     return httpx.AsyncHTTPTransport(
         # pylint: disable=protected-access
         verify=verify,

utils/lib_sxng_container.sh

@@ -114,7 +114,6 @@ container.build() {
         # shellcheck disable=SC2086
         "$container_engine" $params_build_builder \
             --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
-            --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./container/uwsgi.ini)" \
             --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \
             --file="./container/$dockerfile" \
             .
@@ -122,6 +121,8 @@ container.build() {
 
         # shellcheck disable=SC2086
         "$container_engine" $params_build \
+            --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
+            --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./container/config/uwsgi.ini)" \
             --build-arg="GIT_URL=$GIT_URL" \
             --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
             --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \