[mod] container: rework builds (#6720)

Free from ghcr.io/searxng/cache! Now we save and load images without polluting this shared (and useless) layer cache, which also makes it easy to download artifacts for local debugging. I also removed this nonsense about virtualenv normalization, this will be properly fixed by using lockfiles on py dependencies.
2026-09-14 18:26:04 +00:00 · 2026-09-14 16:06:00 +02:00
parent d4ce87c234
commit ef05645f09
4 changed files with 134 additions and 185 deletions
--- a/.github/workflows/container.yml
+++ b/.github/workflows/container.yml
@@ -41,21 +41,10 @@ jobs:
          - runner: ubuntu-26.04-arm
            arch: armv7

-    permissions:
-      packages: write
-
    outputs:
      docker_tag: ${{ steps.build.outputs.docker_tag }}
-      git_url: ${{ steps.build.outputs.git_url }}

    steps:
-      - name: Login to GHCR
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f  # v4.6.0
-        with:
-          registry: "ghcr.io"
-          username: "${{ github.repository_owner }}"
-          password: "${{ secrets.GITHUB_TOKEN }}"
-
      - name: Setup Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
        with:
@@ -93,6 +82,14 @@ jobs:
          OVERRIDE_ARCH: "${{ matrix.arch }}"
        run: make container.build

+      - name: Upload container image
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: container-image-${{ matrix.arch }}
+          path: "/var/tmp/searxng-podman/*.tar"
+          if-no-files-found: error
+          retention-days: 1
+
  test:
    name: Test (${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
@@ -109,13 +106,6 @@ jobs:
            arch: armv7

    steps:
-      - name: Login to GHCR
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f  # v4.6.0
-        with:
-          registry: "ghcr.io"
-          username: "${{ github.repository_owner }}"
-          password: "${{ secrets.GITHUB_TOKEN }}"
-
      - name: Setup QEMU
        uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a  # v4.3.0

@@ -125,10 +115,16 @@ jobs:
          ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
          persist-credentials: "false"

+      - name: Download container image
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          pattern: container-image-${{ matrix.arch }}
+          path: "/var/tmp/searxng-podman"
+          merge-multiple: true
+
      - name: Test
        env:
          OVERRIDE_ARCH: "${{ matrix.arch }}"
-          GIT_URL: "${{ needs.build.outputs.git_url }}"
        run: make container.test

  release:
@@ -163,8 +159,14 @@ jobs:
          ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
          persist-credentials: "false"

+      - name: Download container images
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          pattern: container-image-*
+          path: "/var/tmp/searxng-podman"
+          merge-multiple: true
+
      - name: Release
        env:
-          GIT_URL: "${{ needs.build.outputs.git_url }}"
          DOCKER_TAG: "${{ needs.build.outputs.docker_tag }}"
        run: make container.push
--- a/container/builder.dockerfile
+++ b/container/builder.dockerfile
@@ -3,21 +3,15 @@ FROM docker.io/searxng/base:searxng-builder AS builder
 COPY ./requirements.txt ./requirements-server.txt ./

 ENV UV_NO_MANAGED_PYTHON="true"
-ENV UV_NATIVE_TLS="true"
-
-ARG TIMESTAMP_VENV="0"

 RUN --mount=type=cache,id=uv,target=/root/.cache/uv set -eux -o pipefail; \
-    export SOURCE_DATE_EPOCH="$TIMESTAMP_VENV"; \
    uv venv; \
    uv pip install --requirements ./requirements.txt --requirements ./requirements-server.txt; \
    uv cache prune --ci; \
    find ./.venv/lib/ -type f -exec strip --strip-unneeded {} + || true; \
    find ./.venv/lib/ -type d -name "__pycache__" -exec rm -rf {} +; \
    find ./.venv/lib/ -type f -name "*.pyc" -delete; \
-    python -m compileall -q -f -j 0 --invalidation-mode=unchecked-hash ./.venv/lib/; \
-    find ./.venv/lib/python*/site-packages/*.dist-info/ -type f -name "RECORD" -exec sort -t, -k1,1 -o {} {} \;; \
-    find ./.venv/ -exec touch -h --date="@$TIMESTAMP_VENV" {} +
+    python -m compileall -q -f -j 0 --invalidation-mode=unchecked-hash ./.venv/lib/

 COPY --exclude=./searx/version_frozen.py ./searx/ ./searx/

--- a/container/dist.dockerfile
+++ b/container/dist.dockerfile
@@ -1,7 +1,4 @@
-ARG CONTAINER_IMAGE_ORGANIZATION="searxng"
-ARG CONTAINER_IMAGE_NAME="searxng"
-
-FROM localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder AS builder
+FROM localhost/searxng/searxng:builder AS builder
 FROM docker.io/searxng/base:searxng AS dist

 COPY --chown=977:977 --from=builder /usr/local/searxng/.venv/ ./.venv/
--- a/utils/lib_sxng_container.sh
+++ b/utils/lib_sxng_container.sh
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: AGPL-3.0-or-later

+tmpdir="/var/tmp/searxng-podman/"
+
 container.help() {
    cat <<EOF
 container.:
@@ -8,18 +10,73 @@ container.:
 EOF
 }

-CONTAINER_IMAGE_ORGANIZATION=${GITHUB_REPOSITORY_OWNER:-"searxng"}
-CONTAINER_IMAGE_NAME="searxng"
+container.__get_platform() {
+    local __arch="$1"
+
+    case $__arch in
+        "X64" | "x86_64" | "amd64")
+            arch="amd64"
+            variant=""
+            platform="linux/$arch"
+            ;;
+        "ARM64" | "aarch64" | "arm64")
+            arch="arm64"
+            variant=""
+            platform="linux/$arch"
+            ;;
+        "ARMV7" | "armhf" | "armv7l" | "armv7")
+            arch="arm"
+            variant="v7"
+            platform="linux/$arch/$variant"
+            ;;
+        *)
+            die 1 "unsupported architecture: $__arch"
+            ;;
+    esac
+}
+
+container.__import_oci() {
+    required_commands podman
+
+    local __release_tags=("$DOCKER_TAG" "latest")
+    local __archives=()
+
+    mkdir -p "$tmpdir"
+
+    while IFS= read -r -d '' archive; do
+        __archives+=("$archive")
+    done < <(find "$tmpdir" -maxdepth 1 -type f -name '*.tar' -print0)
+
+    if [ "${#__archives[@]}" -eq 0 ]; then
+        die 1 "no archives found in $tmpdir"
+    fi
+
+    (
+        set -e
+
+        podman manifest rm --ignore "${__release_tags[@]/#/localhost/searxng/searxng:}"
+
+        for tag in "${__release_tags[@]}"; do
+            podman manifest create "localhost/searxng/searxng:$tag"
+
+            for f in "${__archives[@]}"; do
+                podman manifest add "localhost/searxng/searxng:$tag" "oci-archive:$f"
+            done
+        done
+
+        podman manifest inspect "localhost/searxng/searxng:$DOCKER_TAG"
+    )
+    dump_return $?
+}

 container.build() {
-    local parch=${OVERRIDE_ARCH:-$(uname -m)}
+    required_commands git
+
    local container_engine
    local arch
    local variant
    local platform

-    required_commands git
-
    # Check if podman or docker is installed
    if [ "$1" = "podman" ] || [ "$1" = "docker" ]; then
        if ! command -v "$1" &>/dev/null; then
@@ -33,34 +90,13 @@ container.build() {
        elif command -v docker &>/dev/null; then
            container_engine="docker"
        else
-            die 42 "no compatible container engine is installed (podman or docker)"
+            die 42 "no compatible container engine is installed"
        fi
    fi
    info_msg "Selected engine: $container_engine"
    "$container_engine" version

-    # Setup arch specific
-    case $parch in
-        "X64" | "x86_64" | "amd64")
-            arch="amd64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARM64" | "aarch64" | "arm64")
-            arch="arm64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARMV7" | "armhf" | "armv7l" | "armv7")
-            arch="arm"
-            variant="v7"
-            platform="linux/$arch/$variant"
-            ;;
-        *)
-            err_msg "Unsupported architecture; $parch"
-            exit 1
-            ;;
-    esac
+    container.__get_platform "${OVERRIDE_ARCH:-$(uname -m)}"
    info_msg "Selected platform: $platform"

    if [ "$container_engine" = "docker" ] && ! docker buildx version &>/dev/null; then
@@ -73,16 +109,15 @@ container.build() {
        set -e
        pyenv.activate

-        # Check if it is a git repository
        if [ ! -d .git ]; then
-            die 1 "This is not Git repository"
+            die 1 "this is not a Git repository"
        fi

+        # TODO: get current branch
        if ! git remote get-url origin &>/dev/null; then
-            die 1 "There is no remote origin"
+            die 1 "there is no remote origin"
        fi

-        # This is a git repository
        git update-index -q --refresh
        python -m searx.version freeze
        eval "$(python -m searx.version)"
@@ -90,42 +125,30 @@ container.build() {
        info_msg "Set \$DOCKER_TAG: $DOCKER_TAG"
        info_msg "Set \$GIT_URL: $GIT_URL"

-        # change cmp to lockfile when available
-        timestamp_requirements_main=$(git log -1 --format='%ct' ./requirements.txt)
-        timestamp_requirements_server=$(git log -1 --format='%ct' ./requirements-server.txt)
-        if [[ "$timestamp_requirements_main" -ge "$timestamp_requirements_server" ]]; then
-            timestamp_venv="$timestamp_requirements_main"
-        else
-            timestamp_venv="$timestamp_requirements_server"
-        fi
-
        if [ "$container_engine" = "podman" ]; then
-            params_build_builder="build --format=oci --platform=$platform --layers --identity-label=false --timestamp=$timestamp_venv"
-            params_build="build --format=oci --platform=$platform --layers --identity-label=false"
+            params_build_builder="build --format=oci --layers --platform=$platform --identity-label=false"
+            params_build="build --format=oci --layers --platform=$platform --identity-label=false"
        else
            params_build_builder="build --platform=$platform"
            params_build=$params_build_builder
        fi

-        if [ "$GITHUB_ACTIONS" = "true" ]; then
-            params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
-        else
-            params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest"
-            params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG"
+        local_tag_arch="localhost/searxng/searxng:$DOCKER_TAG-$arch$variant"
+        params_build+=" --tag=$local_tag_arch"
+        if [ "$GITHUB_ACTIONS" != "true" ]; then
+            params_build+=" --tag=localhost/searxng/searxng:latest"
+            params_build+=" --tag=localhost/searxng/searxng:$DOCKER_TAG"
        fi

        # shellcheck disable=SC2086
        "$container_engine" $params_build_builder \
-            --build-arg="TIMESTAMP_VENV=$timestamp_venv" \
-            --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \
+            --tag="localhost/searxng/searxng:builder" \
            --file="./container/builder.dockerfile" \
            .
        build_msg CONTAINER "Image \"builder\" built"

        # shellcheck disable=SC2086
        "$container_engine" $params_build \
-            --build-arg="CONTAINER_IMAGE_ORGANIZATION=$CONTAINER_IMAGE_ORGANIZATION" \
-            --build-arg="CONTAINER_IMAGE_NAME=$CONTAINER_IMAGE_NAME" \
            --build-arg="CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            --build-arg="VERSION=$DOCKER_TAG" \
            --build-arg="VCS_URL=$GIT_URL" \
@@ -135,12 +158,17 @@ container.build() {
        build_msg CONTAINER "Image built"

        if [ "$GITHUB_ACTIONS" = "true" ]; then
-            "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
+            required_commands podman
+
+            mkdir -p "$tmpdir"
+            rm -f "$tmpdir"/*.tar
+            image_archive="$tmpdir/image_$arch$variant.tar"
+
+            podman save --format=oci-archive --output="$image_archive" "$local_tag_arch"

            # Output to GHA
            cat <<EOF >>"$GITHUB_OUTPUT"
 docker_tag=$DOCKER_TAG
-git_url=$GIT_URL
 EOF
        fi
    )
@@ -148,50 +176,27 @@ EOF
 }

 container.test() {
-    local parch=${OVERRIDE_ARCH:-$(uname -m)}
-    local arch
-    local variant
-    local platform
-
-    if [ "$GITHUB_ACTIONS" != "true" ]; then
-        die 1 "This command is intended to be run in GitHub Actions"
-    fi
-
    required_commands podman

-    # Setup arch specific
-    case $parch in
-        "X64" | "x86_64" | "amd64")
-            arch="amd64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARM64" | "aarch64" | "arm64")
-            arch="arm64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARMV7" | "armhf" | "armv7l" | "armv7")
-            arch="arm"
-            variant="v7"
-            platform="linux/$arch/$variant"
-            ;;
-        *)
-            err_msg "Unsupported architecture; $parch"
-            exit 1
-            ;;
-    esac
-    build_msg CONTAINER "Selected platform: $platform"
+    local image="$1"
+
+    if [ -z "$image" ]; then
+        image=$(find "$tmpdir" -type f -name '*.tar' -print -quit)
+        if [ -z "$image" ]; then
+            die 1 "no archives found in $tmpdir"
+        fi
+    fi

    (
        set -e

-        podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
+        if [ -f "$image" ]; then
+            image=$(podman load --input="$image" | sed -n 's/^Loaded image: *//p' | tail -n1)
+        fi

-        name="$CONTAINER_IMAGE_NAME-$(date +%N)"
+        name="searxng-$(head -c 32 /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 16)"

-        podman create --name="$name" --rm --timeout=60 --network="host" \
-            "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" >/dev/null
+        podman create --name="$name" --rm --timeout=60 --network="host" "$image" >/dev/null

        podman start "$name" >/dev/null
        podman logs -f "$name" &
@@ -207,81 +212,32 @@ container.test() {
 }

 container.push() {
-    # Architectures on manifest
-    local release_archs=("amd64" "arm64" "armv7")
-
-    local archs=()
-    local variants=()
-    local platforms=()
-
-    if [ "$GITHUB_ACTIONS" != "true" ]; then
-        die 1 "This command is intended to be run in GitHub Actions"
-    fi
-
    required_commands podman

-    for arch in "${release_archs[@]}"; do
-        case $arch in
-            "X64" | "x86_64" | "amd64")
-                archs+=("amd64")
-                variants+=("")
-                platforms+=("linux/${archs[-1]}")
-                ;;
-            "ARM64" | "aarch64" | "arm64")
-                archs+=("arm64")
-                variants+=("")
-                platforms+=("linux/${archs[-1]}")
-                ;;
-            "ARMV7" | "armv7" | "armhf" | "arm")
-                archs+=("arm")
-                variants+=("v7")
-                platforms+=("linux/${archs[-1]}/${variants[-1]}")
-                ;;
-            *)
-                err_msg "Unsupported architecture; $arch"
-                exit 1
-                ;;
-        esac
-    done
+    local release_tags=("$DOCKER_TAG" "latest")
+    local release_registries=("ghcr.io" "docker.io")
+
+    if [ "$GITHUB_ACTIONS" != "true" ]; then
+        die 1 "This command is intended to be run in Actions"
+    fi
+
+    if ! podman manifest exists "localhost/searxng/searxng:${release_tags[0]}" ||
+        ! podman manifest exists "localhost/searxng/searxng:${release_tags[1]}"; then
+        container.__import_oci
+    fi

    (
        set -e

-        # Pull archs
-        for i in "${!archs[@]}"; do
-            podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}"
-        done
-
-        # Manifest tags ("latest" should be the last manifest)
-        release_tags=("$DOCKER_TAG" "latest")
-
-        # Create manifests
-        for tag in "${release_tags[@]}"; do
-            if ! podman manifest exists "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"; then
-                podman manifest create "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"
-            fi
-
-            # Add archs to manifest
-            for i in "${!archs[@]}"; do
-                podman manifest add \
-                    "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \
-                    "containers-storage:ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}"
-            done
-        done
-
        podman image list

-        # Remote registries
-        release_registries=("ghcr.io" "docker.io")
-
-        # Push manifests
        for registry in "${release_registries[@]}"; do
            for tag in "${release_tags[@]}"; do
                build_msg CONTAINER "Pushing manifest $tag to $registry"

-                podman manifest push \
-                    "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \
-                    "docker://$registry/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"
+                podman manifest push --all \
+                    "localhost/searxng/searxng:$tag" \
+                    "docker://$registry/${GITHUB_REPOSITORY_OWNER:-"searxng"}/searxng:$tag"
            done
        done
    )