[fix] ci: cache is not overwritten (#5089)

Due to current limitations of `actions/cache`, the cache cannot be overwritten.
In our case, we need to accumulate cached wheels from different architectures.
To solve this, we simply delete the key before storing the cache again.

.dockerignore

@@ -1,44 +1,5 @@
-*~
+*
-*/*~
-*/*/*~
-*/*/*/*~
-*/*/*/*/*~
 
-# Git
+!container/entrypoint.sh
-.git
+!searx/**
-.gitignore
+!requirements*.txt
-
-# CI
-.codeclimate.yml
-.travis.yml
-.taskcluster.yml
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*/__pycache__/
-*/*/__pycache__/
-*/*/*/__pycache__/
-*.py[cod]
-*/*.py[cod]
-*/*/*.py[cod]
-*/*/*/*.py[cod]
-
-# node_modules
-node_modules/
-*/node_modules/
-*/*/node_modules/
-*/*/*/node_modules/
-*/*/*/*/node_modules/
-
-.tx/
-
-# to sync with .gitignore
-geckodriver.log
-.coverage
-coverage/
-cache/
-build/
-dist/
-local/
-gh-pages/
-*.egg-info/

.github/workflows/cleanup.yml

@@ -34,4 +34,4 @@ jobs:
           image-names: "cache"
           image-tags: "!searxng*"
           cut-off: "1d"
-          keep-n-most-recent: "100"
+          keep-n-most-recent: "30"

.github/workflows/container.yml

@@ -104,6 +104,8 @@ jobs:
     needs: build-base
     strategy:
       fail-fast: false
+      # Faster runners first to cache arch independent wheels
+      max-parallel: 1
       matrix:
         include:
           - arch: amd64
@@ -119,13 +121,12 @@ jobs:
     permissions:
       # Organization GHCR
       packages: write
+      # Clean key cache step
+      actions: write
 
     outputs:
-      version_string: ${{ steps.build.outputs.version_string }}
-      version_tag: ${{ steps.build.outputs.version_tag }}
       docker_tag: ${{ steps.build.outputs.docker_tag }}
       git_url: ${{ steps.build.outputs.git_url }}
-      git_branch: ${{ steps.build.outputs.git_branch }}
 
     steps:
       - name: Setup Python
@@ -145,16 +146,24 @@ jobs:
           restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-"
           path: "./local/"
 
-      - name: Setup cache container mounts
+      - name: Restore cache container mounts
-        uses: actions/cache@v4
+        id: cache-container-mounts
+        uses: actions/cache/restore@v4
         with:
-          # yamllint disable-line rule:line-length
+          key: "container-mounts-${{ hashFiles('./container/*.dockerfile') }}"
-          key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./container/Dockerfile') }}"
+          restore-keys: "container-mounts-"
-          restore-keys: "container-mounts-${{ matrix.arch }}-"
           path: |
             /var/tmp/buildah-cache/
             /var/tmp/buildah-cache-*/
 
+      # https://github.com/actions/cache/pull/1308
+      - if: steps.cache-container-mounts.outputs.cache-hit == 'true'
+        name: Clean key cache container mounts
+        continue-on-error: true
+        env:
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: gh cache delete container-mounts-${{ hashFiles('./container/*.dockerfile') }}
+
       - if: ${{ matrix.emulation }}
         name: Setup QEMU
         uses: docker/setup-qemu-action@v3
@@ -172,6 +181,15 @@ jobs:
           OVERRIDE_ARCH: "${{ matrix.arch }}"
         run: make podman.build
 
+      - if: always()
+        name: Save cache container mounts
+        uses: actions/cache/save@v4
+        with:
+          key: "container-mounts-${{ hashFiles('./container/*.dockerfile') }}"
+          path: |
+            /var/tmp/buildah-cache/
+            /var/tmp/buildah-cache-*/
+
   test:
     name: Test (${{ matrix.arch }})
     runs-on: ${{ matrix.os }}

container/Dockerfile

@@ -1,61 +0,0 @@
-FROM ghcr.io/searxng/base:searxng-builder AS builder
-
-COPY ./requirements*.txt ./
-
-RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \
- && . ./venv/bin/activate \
- && pip install -r requirements.txt \
- && pip install -r requirements-server.txt
-
-COPY ./searx/ ./searx/
-
-ARG TIMESTAMP_SETTINGS="0"
-
-RUN python -m compileall -q searx \
- && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \
- && find ./searx/static \
-    \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" -o -name "*.ttf" -o -name "*.eot" \) \
-    -type f -exec gzip -9 -k {} + -exec brotli --best {} +
-
-FROM ghcr.io/searxng/base:searxng AS dist
-
-ARG LABEL_DATE="0001-01-01T00:00:00Z"
-ARG GIT_URL="unspecified"
-ARG SEARXNG_GIT_VERSION="unspecified"
-ARG LABEL_VCS_REF="unspecified"
-ARG LABEL_VCS_URL="unspecified"
-
-COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/
-COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/
-COPY --chown=searxng:searxng ./container/template/ ./.template/
-COPY --chown=searxng:searxng ./container/entrypoint.sh ./entrypoint.sh
-
-LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \
-      org.opencontainers.image.created="$LABEL_DATE" \
-      org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \
-      org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \
-      org.opencontainers.image.licenses="AGPL-3.0-or-later" \
-      org.opencontainers.image.revision="$LABEL_VCS_REF" \
-      org.opencontainers.image.source="$LABEL_VCS_URL" \
-      org.opencontainers.image.title="searxng" \
-      org.opencontainers.image.url="$LABEL_VCS_URL" \
-      org.opencontainers.image.version="$SEARXNG_GIT_VERSION"
-
-ENV SEARXNG_VERSION="$SEARXNG_GIT_VERSION" \
-    SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \
-    GRANIAN_PROCESS_NAME="searxng" \
-    GRANIAN_INTERFACE="wsgi" \
-    GRANIAN_HOST="::" \
-    GRANIAN_PORT="8080" \
-    GRANIAN_WEBSOCKETS="false" \
-    GRANIAN_LOOP="uvloop" \
-    GRANIAN_BLOCKING_THREADS="4" \
-    GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
-    GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m"
-
-VOLUME $CONFIG_PATH
-VOLUME $DATA_PATH
-
-EXPOSE 8080
-
-ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]

container/builder.dockerfile

@@ -0,0 +1,24 @@
+FROM ghcr.io/searxng/base:searxng-builder AS builder
+
+COPY ./requirements*.txt ./
+
+RUN --mount=type=cache,id=pip,target=/root/.cache/pip set -eux; \
+    python -m venv ./.venv/; \
+    . ./.venv/bin/activate; \
+    pip install -r ./requirements.txt -r ./requirements-server.txt
+
+COPY ./searx/ ./searx/
+
+ARG TIMESTAMP_SETTINGS="0"
+
+RUN set -eux; \
+    python -m compileall -q ./searx/; \
+    touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml; \
+    find ./searx/static/ -type f \
+        \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" \) \
+        -exec gzip -9 -k {} + \
+        -exec brotli -9 -k {} + \
+        -exec gzip --test {}.gz + \
+        -exec brotli --test {}.br +; \
+    # Move always changing files to /usr/local/searxng/
+    mv ./searx/version_frozen.py ./

container/dist.dockerfile

@@ -0,0 +1,44 @@
+FROM ghcr.io/searxng/base:searxng AS dist
+
+ARG CONTAINER_IMAGE_ORGANIZATION="searxng"
+ARG CONTAINER_IMAGE_NAME="searxng"
+
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/.venv/ ./.venv/
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/searx/ ./searx/
+COPY --chown=searxng:searxng ./container/ ./
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/version_frozen.py ./searx/
+
+ARG CREATED="0001-01-01T00:00:00Z"
+ARG VERSION="unknown"
+ARG VCS_URL="unknown"
+ARG VCS_REVISION="unknown"
+
+LABEL org.opencontainers.image.created="$CREATED" \
+      org.opencontainers.image.description="SearXNG is a metasearch engine. Users are neither tracked nor profiled." \
+      org.opencontainers.image.documentation="https://docs.searxng.org/admin/installation-docker" \
+      org.opencontainers.image.licenses="AGPL-3.0-or-later" \
+      org.opencontainers.image.revision="$VCS_REVISION" \
+      org.opencontainers.image.source="$VCS_URL" \
+      org.opencontainers.image.title="SearXNG" \
+      org.opencontainers.image.url="https://searxng.org" \
+      org.opencontainers.image.version="$VERSION"
+
+ENV SEARXNG_VERSION="$VERSION" \
+    SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \
+    GRANIAN_PROCESS_NAME="searxng" \
+    GRANIAN_INTERFACE="wsgi" \
+    GRANIAN_HOST="::" \
+    GRANIAN_PORT="8080" \
+    GRANIAN_WEBSOCKETS="false" \
+    GRANIAN_LOOP="uvloop" \
+    GRANIAN_BLOCKING_THREADS="4" \
+    GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
+    GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m"
+
+# "*_PATH" ENVs are defined in base images
+VOLUME $CONFIG_PATH
+VOLUME $DATA_PATH
+
+EXPOSE 8080
+
+ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]

container/entrypoint.sh

@@ -127,4 +127,4 @@ volume_handler "$DATA_PATH"
 # Check for files
 config_handler "$SEARXNG_SETTINGS_PATH" "/usr/local/searxng/searx/settings.yml"
 
-exec /usr/local/searxng/venv/bin/granian searx.webapp:app
+exec /usr/local/searxng/.venv/bin/granian searx.webapp:app

docs/admin/installation-docker.rst

@@ -80,7 +80,7 @@ Pull the latest image:
 
 .. code:: sh
 
-   $ docker pull docker.io/searxng/searxng:2025.6.3-b73ac81
+   $ docker pull docker.io/searxng/searxng:2025.8.1-3d96414
 
 .. _Container instancing:
 
@@ -119,14 +119,14 @@ List running containers:
 
    $ docker container list
    CONTAINER ID  IMAGE  ...  CREATED        PORTS                   NAMES
-   37f6487c8703  ...    ...  3 minutes ago  0.0.0.0:8888->8080/tcp  searxng
+   1af574997e63  ...    ...  3 minutes ago  0.0.0.0:8888->8080/tcp  searxng
 
 Access the container shell (troubleshooting):
 
 .. code:: sh
 
    $ docker container exec -it --user root searxng /bin/sh -l
-   37f6487c8703:/usr/local/searxng#
+   1af574997e63:/usr/local/searxng#
 
 Stop and remove the container:
 
@@ -182,9 +182,9 @@ container images are not officially supported):
    $ make container
 
    $ docker images
-   REPOSITORY                 TAG                       IMAGE ID      CREATED         SIZE
+   REPOSITORY                 TAG                 IMAGE ID  CREATED             SIZE
-   localhost/searxng/searxng  latest                    b14e256bfc36  14 seconds ago  201 MB
+   localhost/searxng/searxng  2025.8.1-3d96414    ...       About a minute ago  183 MB
-   localhost/searxng/searxng  2025.5.1-b653119ab-dirty  b14e256bfc36  14 seconds ago  201 MB
+   localhost/searxng/searxng  latest              ...       About a minute ago  183 MB
-   localhost/searxng/searxng  builder                   7f334c752b41  20 seconds ago  765 MB
+   localhost/searxng/searxng  builder             ...       About a minute ago  524 MB
-   ghcr.io/searxng/base       searxng-builder           7d6b8a1bed4a  20 hours ago    625 MB
+   ghcr.io/searxng/base       searxng-builder     ...       2 days ago          378 MB
-   ghcr.io/searxng/base       searxng                   29baf9ef13ef  20 hours ago    62.5 MB
+   ghcr.io/searxng/base       searxng             ...       2 days ago          42.2 MB

searx/version.py

@@ -134,9 +134,14 @@ DOCKER_TAG = "{DOCKER_TAG}"
 GIT_URL = "{GIT_URL}"
 GIT_BRANCH = "{GIT_BRANCH}"
 """
-        with open(os.path.join(os.path.dirname(__file__), "version_frozen.py"), "w", encoding="utf8") as f:
+        path = os.path.join(os.path.dirname(__file__), "version_frozen.py")
+        with open(path, "w", encoding="utf8") as f:
             f.write(python_code)
             print(f"{f.name} created")
+
+        # set file timestamp to commit timestamp
+        commit_timestamp = int(subprocess_run("git show -s --format=%ct"))
+        os.utime(path, (commit_timestamp, commit_timestamp))
     else:
         # output shell code to set the variables
         # usage: eval "$(python -m searx.version)"

utils/lib_sxng_container.sh

@@ -14,7 +14,6 @@ CONTAINER_IMAGE_NAME="searxng"
 container.build() {
     local parch=${OVERRIDE_ARCH:-$(uname -m)}
     local container_engine
-    local dockerfile
     local arch
     local variant
     local platform
@@ -42,19 +41,16 @@ container.build() {
     # Setup arch specific
     case $parch in
         "X64" | "x86_64" | "amd64")
-            dockerfile="Dockerfile"
             arch="amd64"
             variant=""
             platform="linux/$arch"
             ;;
         "ARM64" | "aarch64" | "arm64")
-            dockerfile="Dockerfile"
             arch="arm64"
             variant=""
             platform="linux/$arch"
             ;;
         "ARMV7" | "armhf" | "armv7l" | "armv7")
-            dockerfile="Dockerfile"
             arch="arm"
             variant="v7"
             platform="linux/$arch/$variant"
@@ -86,27 +82,20 @@ container.build() {
         python -m searx.version freeze
         eval "$(python -m searx.version)"
 
-        info_msg "Set \$VERSION_STRING: $VERSION_STRING"
-        info_msg "Set \$VERSION_TAG: $VERSION_TAG"
         info_msg "Set \$DOCKER_TAG: $DOCKER_TAG"
         info_msg "Set \$GIT_URL: $GIT_URL"
-        info_msg "Set \$GIT_BRANCH: $GIT_BRANCH"
 
         if [ "$container_engine" = "podman" ]; then
-            params_build_builder="build --format=oci --platform=$platform --target=builder --layers --identity-label=false"
+            params_build_builder="build --format=oci --platform=$platform --layers --identity-label=false"
-            params_build="build --format=oci --platform=$platform --layers --squash-all --omit-history --identity-label=false"
+            params_build=$params_build_builder
         else
-            params_build_builder="build --platform=$platform --target=builder"
+            params_build_builder="build --platform=$platform"
-            params_build="build --platform=$platform --squash"
+            params_build=$params_build_builder
         fi
 
         if [ "$GITHUB_ACTIONS" = "true" ]; then
-            params_build_builder+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache"
-
-            # Tags
             params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
         else
-            # Tags
             params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest"
             params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG"
         fi
@@ -115,19 +104,19 @@ container.build() {
         "$container_engine" $params_build_builder \
             --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
             --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \
-            --file="./container/$dockerfile" \
+            --file="./container/builder.dockerfile" \
             .
         build_msg CONTAINER "Image \"builder\" built"
 
         # shellcheck disable=SC2086
         "$container_engine" $params_build \
-            --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
+            --build-arg="CONTAINER_IMAGE_ORGANIZATION=$CONTAINER_IMAGE_ORGANIZATION" \
-            --build-arg="GIT_URL=$GIT_URL" \
+            --build-arg="CONTAINER_IMAGE_NAME=$CONTAINER_IMAGE_NAME" \
-            --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
+            --build-arg="CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-            --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            --build-arg="VERSION=$DOCKER_TAG" \
-            --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
+            --build-arg="VCS_URL=$GIT_URL" \
-            --build-arg="LABEL_VCS_URL=$GIT_URL" \
+            --build-arg="VCS_REVISION=$(git rev-parse HEAD)" \
-            --file="./container/$dockerfile" \
+            --file="./container/dist.dockerfile" \
             .
         build_msg CONTAINER "Image built"
 
@@ -136,11 +125,8 @@ container.build() {
 
             # Output to GHA
             cat <<EOF >>"$GITHUB_OUTPUT"
-version_string=$VERSION_STRING
-version_tag=$VERSION_TAG
 docker_tag=$DOCKER_TAG
 git_url=$GIT_URL
-git_branch=$GIT_BRANCH
 EOF
         fi
     )