[fix] ci: cache is not overwritten (#5089 )

Due to current limitations of `actions/cache`, the cache cannot be overwritten. In our case, we need to accumulate cached wheels from different architectures. To solve this, we simply delete the key before storing the cache again.
[enh] container: tidy builds (#5086 )
2026-08-01 21:01:24 +00:00 · 2025-08-07 16:53:37 +02:00 · 2025-08-07 10:46:26 +02:00
11 changed files with 127 additions and 150 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,44 +1,5 @@
-*~
-*/*~
-*/*/*~
-*/*/*/*~
-*/*/*/*/*~
+*

-# Git
-.git
-.gitignore
-
-# CI
-.codeclimate.yml
-.travis.yml
-.taskcluster.yml
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*/__pycache__/
-*/*/__pycache__/
-*/*/*/__pycache__/
-*.py[cod]
-*/*.py[cod]
-*/*/*.py[cod]
-*/*/*/*.py[cod]
-
-# node_modules
-node_modules/
-*/node_modules/
-*/*/node_modules/
-*/*/*/node_modules/
-*/*/*/*/node_modules/
-
-.tx/
-
-# to sync with .gitignore
-geckodriver.log
-.coverage
-coverage/
-cache/
-build/
-dist/
-local/
-gh-pages/
-*.egg-info/
+!container/entrypoint.sh
+!searx/**
+!requirements*.txt
--- a/.github/workflows/cleanup.yml
+++ b/.github/workflows/cleanup.yml
@@ -34,4 +34,4 @@ jobs:
          image-names: "cache"
          image-tags: "!searxng*"
          cut-off: "1d"
-          keep-n-most-recent: "100"
+          keep-n-most-recent: "30"
--- a/.github/workflows/container.yml
+++ b/.github/workflows/container.yml
@@ -104,6 +104,8 @@ jobs:
    needs: build-base
    strategy:
      fail-fast: false
+      # Faster runners first to cache arch independent wheels
+      max-parallel: 1
      matrix:
        include:
          - arch: amd64
@@ -119,13 +121,12 @@ jobs:
    permissions:
      # Organization GHCR
      packages: write
+      # Clean key cache step
+      actions: write

    outputs:
-      version_string: ${{ steps.build.outputs.version_string }}
-      version_tag: ${{ steps.build.outputs.version_tag }}
      docker_tag: ${{ steps.build.outputs.docker_tag }}
      git_url: ${{ steps.build.outputs.git_url }}
-      git_branch: ${{ steps.build.outputs.git_branch }}

    steps:
      - name: Setup Python
@@ -145,16 +146,24 @@ jobs:
          restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-"
          path: "./local/"

-      - name: Setup cache container mounts
-        uses: actions/cache@v4
+      - name: Restore cache container mounts
+        id: cache-container-mounts
+        uses: actions/cache/restore@v4
        with:
-          # yamllint disable-line rule:line-length
-          key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./container/Dockerfile') }}"
-          restore-keys: "container-mounts-${{ matrix.arch }}-"
+          key: "container-mounts-${{ hashFiles('./container/*.dockerfile') }}"
+          restore-keys: "container-mounts-"
          path: |
            /var/tmp/buildah-cache/
            /var/tmp/buildah-cache-*/

+      # https://github.com/actions/cache/pull/1308
+      - if: steps.cache-container-mounts.outputs.cache-hit == 'true'
+        name: Clean key cache container mounts
+        continue-on-error: true
+        env:
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: gh cache delete container-mounts-${{ hashFiles('./container/*.dockerfile') }}
+
      - if: ${{ matrix.emulation }}
        name: Setup QEMU
        uses: docker/setup-qemu-action@v3
@@ -172,6 +181,15 @@ jobs:
          OVERRIDE_ARCH: "${{ matrix.arch }}"
        run: make podman.build

+      - if: always()
+        name: Save cache container mounts
+        uses: actions/cache/save@v4
+        with:
+          key: "container-mounts-${{ hashFiles('./container/*.dockerfile') }}"
+          path: |
+            /var/tmp/buildah-cache/
+            /var/tmp/buildah-cache-*/
+
  test:
    name: Test (${{ matrix.arch }})
    runs-on: ${{ matrix.os }}
--- a/container/Dockerfile
+++ b/container/Dockerfile
@@ -1,61 +0,0 @@
-FROM ghcr.io/searxng/base:searxng-builder AS builder
-
-COPY ./requirements*.txt ./
-
-RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \
- && . ./venv/bin/activate \
- && pip install -r requirements.txt \
- && pip install -r requirements-server.txt
-
-COPY ./searx/ ./searx/
-
-ARG TIMESTAMP_SETTINGS="0"
-
-RUN python -m compileall -q searx \
- && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \
- && find ./searx/static \
-    \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" -o -name "*.ttf" -o -name "*.eot" \) \
-    -type f -exec gzip -9 -k {} + -exec brotli --best {} +
-
-FROM ghcr.io/searxng/base:searxng AS dist
-
-ARG LABEL_DATE="0001-01-01T00:00:00Z"
-ARG GIT_URL="unspecified"
-ARG SEARXNG_GIT_VERSION="unspecified"
-ARG LABEL_VCS_REF="unspecified"
-ARG LABEL_VCS_URL="unspecified"
-
-COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/
-COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/
-COPY --chown=searxng:searxng ./container/template/ ./.template/
-COPY --chown=searxng:searxng ./container/entrypoint.sh ./entrypoint.sh
-
-LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \
-      org.opencontainers.image.created="$LABEL_DATE" \
-      org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \
-      org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \
-      org.opencontainers.image.licenses="AGPL-3.0-or-later" \
-      org.opencontainers.image.revision="$LABEL_VCS_REF" \
-      org.opencontainers.image.source="$LABEL_VCS_URL" \
-      org.opencontainers.image.title="searxng" \
-      org.opencontainers.image.url="$LABEL_VCS_URL" \
-      org.opencontainers.image.version="$SEARXNG_GIT_VERSION"
-
-ENV SEARXNG_VERSION="$SEARXNG_GIT_VERSION" \
-    SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \
-    GRANIAN_PROCESS_NAME="searxng" \
-    GRANIAN_INTERFACE="wsgi" \
-    GRANIAN_HOST="::" \
-    GRANIAN_PORT="8080" \
-    GRANIAN_WEBSOCKETS="false" \
-    GRANIAN_LOOP="uvloop" \
-    GRANIAN_BLOCKING_THREADS="4" \
-    GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
-    GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m"
-
-VOLUME $CONFIG_PATH
-VOLUME $DATA_PATH
-
-EXPOSE 8080
-
-ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]
--- a/container/builder.dockerfile
+++ b/container/builder.dockerfile
@@ -0,0 +1,24 @@
+FROM ghcr.io/searxng/base:searxng-builder AS builder
+
+COPY ./requirements*.txt ./
+
+RUN --mount=type=cache,id=pip,target=/root/.cache/pip set -eux; \
+    python -m venv ./.venv/; \
+    . ./.venv/bin/activate; \
+    pip install -r ./requirements.txt -r ./requirements-server.txt
+
+COPY ./searx/ ./searx/
+
+ARG TIMESTAMP_SETTINGS="0"
+
+RUN set -eux; \
+    python -m compileall -q ./searx/; \
+    touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml; \
+    find ./searx/static/ -type f \
+        \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" \) \
+        -exec gzip -9 -k {} + \
+        -exec brotli -9 -k {} + \
+        -exec gzip --test {}.gz + \
+        -exec brotli --test {}.br +; \
+    # Move always changing files to /usr/local/searxng/
+    mv ./searx/version_frozen.py ./
--- a/container/dist.dockerfile
+++ b/container/dist.dockerfile
@@ -0,0 +1,44 @@
+FROM ghcr.io/searxng/base:searxng AS dist
+
+ARG CONTAINER_IMAGE_ORGANIZATION="searxng"
+ARG CONTAINER_IMAGE_NAME="searxng"
+
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/.venv/ ./.venv/
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/searx/ ./searx/
+COPY --chown=searxng:searxng ./container/ ./
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/version_frozen.py ./searx/
+
+ARG CREATED="0001-01-01T00:00:00Z"
+ARG VERSION="unknown"
+ARG VCS_URL="unknown"
+ARG VCS_REVISION="unknown"
+
+LABEL org.opencontainers.image.created="$CREATED" \
+      org.opencontainers.image.description="SearXNG is a metasearch engine. Users are neither tracked nor profiled." \
+      org.opencontainers.image.documentation="https://docs.searxng.org/admin/installation-docker" \
+      org.opencontainers.image.licenses="AGPL-3.0-or-later" \
+      org.opencontainers.image.revision="$VCS_REVISION" \
+      org.opencontainers.image.source="$VCS_URL" \
+      org.opencontainers.image.title="SearXNG" \
+      org.opencontainers.image.url="https://searxng.org" \
+      org.opencontainers.image.version="$VERSION"
+
+ENV SEARXNG_VERSION="$VERSION" \
+    SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \
+    GRANIAN_PROCESS_NAME="searxng" \
+    GRANIAN_INTERFACE="wsgi" \
+    GRANIAN_HOST="::" \
+    GRANIAN_PORT="8080" \
+    GRANIAN_WEBSOCKETS="false" \
+    GRANIAN_LOOP="uvloop" \
+    GRANIAN_BLOCKING_THREADS="4" \
+    GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
+    GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m"
+
+# "*_PATH" ENVs are defined in base images
+VOLUME $CONFIG_PATH
+VOLUME $DATA_PATH
+
+EXPOSE 8080
+
+ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]
--- a/container/entrypoint.sh
+++ b/container/entrypoint.sh
@@ -127,4 +127,4 @@ volume_handler "$DATA_PATH"
 # Check for files
 config_handler "$SEARXNG_SETTINGS_PATH" "/usr/local/searxng/searx/settings.yml"

-exec /usr/local/searxng/venv/bin/granian searx.webapp:app
+exec /usr/local/searxng/.venv/bin/granian searx.webapp:app
--- a/container/template/.empty
+++ b/container/template/.empty
--- a/docs/admin/installation-docker.rst
+++ b/docs/admin/installation-docker.rst
@@ -80,7 +80,7 @@ Pull the latest image:

 .. code:: sh

-   $ docker pull docker.io/searxng/searxng:2025.6.3-b73ac81
+   $ docker pull docker.io/searxng/searxng:2025.8.1-3d96414

 .. _Container instancing:

@@ -119,14 +119,14 @@ List running containers:

   $ docker container list
   CONTAINER ID  IMAGE  ...  CREATED        PORTS                   NAMES
-   37f6487c8703  ...    ...  3 minutes ago  0.0.0.0:8888->8080/tcp  searxng
+   1af574997e63  ...    ...  3 minutes ago  0.0.0.0:8888->8080/tcp  searxng

 Access the container shell (troubleshooting):

 .. code:: sh

   $ docker container exec -it --user root searxng /bin/sh -l
-   37f6487c8703:/usr/local/searxng#
+   1af574997e63:/usr/local/searxng#

 Stop and remove the container:

@@ -182,9 +182,9 @@ container images are not officially supported):
   $ make container

   $ docker images
-   REPOSITORY                 TAG                       IMAGE ID      CREATED         SIZE
-   localhost/searxng/searxng  latest                    b14e256bfc36  14 seconds ago  201 MB
-   localhost/searxng/searxng  2025.5.1-b653119ab-dirty  b14e256bfc36  14 seconds ago  201 MB
-   localhost/searxng/searxng  builder                   7f334c752b41  20 seconds ago  765 MB
-   ghcr.io/searxng/base       searxng-builder           7d6b8a1bed4a  20 hours ago    625 MB
-   ghcr.io/searxng/base       searxng                   29baf9ef13ef  20 hours ago    62.5 MB
+   REPOSITORY                 TAG                 IMAGE ID  CREATED             SIZE
+   localhost/searxng/searxng  2025.8.1-3d96414    ...       About a minute ago  183 MB
+   localhost/searxng/searxng  latest              ...       About a minute ago  183 MB
+   localhost/searxng/searxng  builder             ...       About a minute ago  524 MB
+   ghcr.io/searxng/base       searxng-builder     ...       2 days ago          378 MB
+   ghcr.io/searxng/base       searxng             ...       2 days ago          42.2 MB
--- a/searx/version.py
+++ b/searx/version.py
@@ -134,9 +134,14 @@ DOCKER_TAG = "{DOCKER_TAG}"
 GIT_URL = "{GIT_URL}"
 GIT_BRANCH = "{GIT_BRANCH}"
 """
-        with open(os.path.join(os.path.dirname(__file__), "version_frozen.py"), "w", encoding="utf8") as f:
+        path = os.path.join(os.path.dirname(__file__), "version_frozen.py")
+        with open(path, "w", encoding="utf8") as f:
            f.write(python_code)
            print(f"{f.name} created")
+
+        # set file timestamp to commit timestamp
+        commit_timestamp = int(subprocess_run("git show -s --format=%ct"))
+        os.utime(path, (commit_timestamp, commit_timestamp))
    else:
        # output shell code to set the variables
        # usage: eval "$(python -m searx.version)"
--- a/utils/lib_sxng_container.sh
+++ b/utils/lib_sxng_container.sh
@@ -14,7 +14,6 @@ CONTAINER_IMAGE_NAME="searxng"
 container.build() {
    local parch=${OVERRIDE_ARCH:-$(uname -m)}
    local container_engine
-    local dockerfile
    local arch
    local variant
    local platform
@@ -42,19 +41,16 @@ container.build() {
    # Setup arch specific
    case $parch in
        "X64" | "x86_64" | "amd64")
-            dockerfile="Dockerfile"
            arch="amd64"
            variant=""
            platform="linux/$arch"
            ;;
        "ARM64" | "aarch64" | "arm64")
-            dockerfile="Dockerfile"
            arch="arm64"
            variant=""
            platform="linux/$arch"
            ;;
        "ARMV7" | "armhf" | "armv7l" | "armv7")
-            dockerfile="Dockerfile"
            arch="arm"
            variant="v7"
            platform="linux/$arch/$variant"
@@ -86,27 +82,20 @@ container.build() {
        python -m searx.version freeze
        eval "$(python -m searx.version)"

-        info_msg "Set \$VERSION_STRING: $VERSION_STRING"
-        info_msg "Set \$VERSION_TAG: $VERSION_TAG"
        info_msg "Set \$DOCKER_TAG: $DOCKER_TAG"
        info_msg "Set \$GIT_URL: $GIT_URL"
-        info_msg "Set \$GIT_BRANCH: $GIT_BRANCH"

        if [ "$container_engine" = "podman" ]; then
-            params_build_builder="build --format=oci --platform=$platform --target=builder --layers --identity-label=false"
-            params_build="build --format=oci --platform=$platform --layers --squash-all --omit-history --identity-label=false"
+            params_build_builder="build --format=oci --platform=$platform --layers --identity-label=false"
+            params_build=$params_build_builder
        else
-            params_build_builder="build --platform=$platform --target=builder"
-            params_build="build --platform=$platform --squash"
+            params_build_builder="build --platform=$platform"
+            params_build=$params_build_builder
        fi

        if [ "$GITHUB_ACTIONS" = "true" ]; then
-            params_build_builder+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache"
-
-            # Tags
            params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
        else
-            # Tags
            params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest"
            params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG"
        fi
@@ -115,19 +104,19 @@ container.build() {
        "$container_engine" $params_build_builder \
            --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
            --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \
-            --file="./container/$dockerfile" \
+            --file="./container/builder.dockerfile" \
            .
        build_msg CONTAINER "Image \"builder\" built"

        # shellcheck disable=SC2086
        "$container_engine" $params_build \
-            --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
-            --build-arg="GIT_URL=$GIT_URL" \
-            --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
-            --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-            --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
-            --build-arg="LABEL_VCS_URL=$GIT_URL" \
-            --file="./container/$dockerfile" \
+            --build-arg="CONTAINER_IMAGE_ORGANIZATION=$CONTAINER_IMAGE_ORGANIZATION" \
+            --build-arg="CONTAINER_IMAGE_NAME=$CONTAINER_IMAGE_NAME" \
+            --build-arg="CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            --build-arg="VERSION=$DOCKER_TAG" \
+            --build-arg="VCS_URL=$GIT_URL" \
+            --build-arg="VCS_REVISION=$(git rev-parse HEAD)" \
+            --file="./container/dist.dockerfile" \
            .
        build_msg CONTAINER "Image built"

@@ -136,11 +125,8 @@ container.build() {

            # Output to GHA
            cat <<EOF >>"$GITHUB_OUTPUT"
-version_string=$VERSION_STRING
-version_tag=$VERSION_TAG
 docker_tag=$DOCKER_TAG
 git_url=$GIT_URL
-git_branch=$GIT_BRANCH
 EOF
        fi
    )