[mod] container: rework builds (#6720)

Free from ghcr.io/searxng/cache! Now we save and load images without polluting
this shared (and useless) layer cache, which also makes it easy to download
artifacts for local debugging.

I also removed this nonsense about virtualenv normalization, this will be
properly fixed by using lockfiles on py dependencies.

.github/workflows/container.yml

@@ -41,21 +41,10 @@ jobs:
           - runner: ubuntu-26.04-arm
             arch: armv7
 
-    permissions:
-      packages: write
-
     outputs:
       docker_tag: ${{ steps.build.outputs.docker_tag }}
-      git_url: ${{ steps.build.outputs.git_url }}
 
     steps:
-      - name: Login to GHCR
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f  # v4.6.0
-        with:
-          registry: "ghcr.io"
-          username: "${{ github.repository_owner }}"
-          password: "${{ secrets.GITHUB_TOKEN }}"
-
       - name: Setup Python
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
         with:
@@ -93,6 +82,14 @@ jobs:
           OVERRIDE_ARCH: "${{ matrix.arch }}"
         run: make container.build
 
+      - name: Upload container image
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: container-image-${{ matrix.arch }}
+          path: "/var/tmp/searxng-podman/*.tar"
+          if-no-files-found: error
+          retention-days: 1
+
   test:
     name: Test (${{ matrix.arch }})
     runs-on: ${{ matrix.runner }}
@@ -109,13 +106,6 @@ jobs:
             arch: armv7
 
     steps:
-      - name: Login to GHCR
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f  # v4.6.0
-        with:
-          registry: "ghcr.io"
-          username: "${{ github.repository_owner }}"
-          password: "${{ secrets.GITHUB_TOKEN }}"
-
       - name: Setup QEMU
         uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a  # v4.3.0
 
@@ -125,10 +115,16 @@ jobs:
           ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
           persist-credentials: "false"
 
+      - name: Download container image
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          pattern: container-image-${{ matrix.arch }}
+          path: "/var/tmp/searxng-podman"
+          merge-multiple: true
+
       - name: Test
         env:
           OVERRIDE_ARCH: "${{ matrix.arch }}"
-          GIT_URL: "${{ needs.build.outputs.git_url }}"
         run: make container.test
 
   release:
@@ -163,8 +159,14 @@ jobs:
           ref: "${{ github.event.workflow_run.head_sha || github.sha }}"
           persist-credentials: "false"
 
+      - name: Download container images
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          pattern: container-image-*
+          path: "/var/tmp/searxng-podman"
+          merge-multiple: true
+
       - name: Release
         env:
-          GIT_URL: "${{ needs.build.outputs.git_url }}"
           DOCKER_TAG: "${{ needs.build.outputs.docker_tag }}"
         run: make container.push

container/builder.dockerfile

@@ -3,21 +3,15 @@ FROM docker.io/searxng/base:searxng-builder AS builder
 COPY ./requirements.txt ./requirements-server.txt ./
 
 ENV UV_NO_MANAGED_PYTHON="true"
-ENV UV_NATIVE_TLS="true"
-
-ARG TIMESTAMP_VENV="0"
 
 RUN --mount=type=cache,id=uv,target=/root/.cache/uv set -eux -o pipefail; \
-    export SOURCE_DATE_EPOCH="$TIMESTAMP_VENV"; \
     uv venv; \
     uv pip install --requirements ./requirements.txt --requirements ./requirements-server.txt; \
     uv cache prune --ci; \
     find ./.venv/lib/ -type f -exec strip --strip-unneeded {} + || true; \
     find ./.venv/lib/ -type d -name "__pycache__" -exec rm -rf {} +; \
     find ./.venv/lib/ -type f -name "*.pyc" -delete; \
-    python -m compileall -q -f -j 0 --invalidation-mode=unchecked-hash ./.venv/lib/; \
+    python -m compileall -q -f -j 0 --invalidation-mode=unchecked-hash ./.venv/lib/
-    find ./.venv/lib/python*/site-packages/*.dist-info/ -type f -name "RECORD" -exec sort -t, -k1,1 -o {} {} \;; \
-    find ./.venv/ -exec touch -h --date="@$TIMESTAMP_VENV" {} +
 
 COPY --exclude=./searx/version_frozen.py ./searx/ ./searx/
 

container/dist.dockerfile

@@ -1,7 +1,4 @@
-ARG CONTAINER_IMAGE_ORGANIZATION="searxng"
+FROM localhost/searxng/searxng:builder AS builder
-ARG CONTAINER_IMAGE_NAME="searxng"
-
-FROM localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder AS builder
 FROM docker.io/searxng/base:searxng AS dist
 
 COPY --chown=977:977 --from=builder /usr/local/searxng/.venv/ ./.venv/

utils/lib_sxng_container.sh

@@ -1,25 +1,82 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+tmpdir="/var/tmp/searxng-podman/"
+
 container.help() {
     cat <<EOF
 container.:
-  build     : build container image
+  build   : build container image
 EOF
 }
 
-CONTAINER_IMAGE_ORGANIZATION=${GITHUB_REPOSITORY_OWNER:-"searxng"}
+container.__get_platform() {
-CONTAINER_IMAGE_NAME="searxng"
+    local __arch="$1"
+
+    case $__arch in
+        "X64" | "x86_64" | "amd64")
+            arch="amd64"
+            variant=""
+            platform="linux/$arch"
+            ;;
+        "ARM64" | "aarch64" | "arm64")
+            arch="arm64"
+            variant=""
+            platform="linux/$arch"
+            ;;
+        "ARMV7" | "armhf" | "armv7l" | "armv7")
+            arch="arm"
+            variant="v7"
+            platform="linux/$arch/$variant"
+            ;;
+        *)
+            die 1 "unsupported architecture: $__arch"
+            ;;
+    esac
+}
+
+container.__import_oci() {
+    required_commands podman
+
+    local __release_tags=("$DOCKER_TAG" "latest")
+    local __archives=()
+
+    mkdir -p "$tmpdir"
+
+    while IFS= read -r -d '' archive; do
+        __archives+=("$archive")
+    done < <(find "$tmpdir" -maxdepth 1 -type f -name '*.tar' -print0)
+
+    if [ "${#__archives[@]}" -eq 0 ]; then
+        die 1 "no archives found in $tmpdir"
+    fi
+
+    (
+        set -e
+
+        podman manifest rm --ignore "${__release_tags[@]/#/localhost/searxng/searxng:}"
+
+        for tag in "${__release_tags[@]}"; do
+            podman manifest create "localhost/searxng/searxng:$tag"
+
+            for f in "${__archives[@]}"; do
+                podman manifest add "localhost/searxng/searxng:$tag" "oci-archive:$f"
+            done
+        done
+
+        podman manifest inspect "localhost/searxng/searxng:$DOCKER_TAG"
+    )
+    dump_return $?
+}
 
 container.build() {
-    local parch=${OVERRIDE_ARCH:-$(uname -m)}
+    required_commands git
+
     local container_engine
     local arch
     local variant
     local platform
 
-    required_commands git
-
     # Check if podman or docker is installed
     if [ "$1" = "podman" ] || [ "$1" = "docker" ]; then
         if ! command -v "$1" &>/dev/null; then
@@ -33,34 +90,13 @@ container.build() {
         elif command -v docker &>/dev/null; then
             container_engine="docker"
         else
-            die 42 "no compatible container engine is installed (podman or docker)"
+            die 42 "no compatible container engine is installed"
         fi
     fi
     info_msg "Selected engine: $container_engine"
     "$container_engine" version
 
-    # Setup arch specific
+    container.__get_platform "${OVERRIDE_ARCH:-$(uname -m)}"
-    case $parch in
-        "X64" | "x86_64" | "amd64")
-            arch="amd64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARM64" | "aarch64" | "arm64")
-            arch="arm64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARMV7" | "armhf" | "armv7l" | "armv7")
-            arch="arm"
-            variant="v7"
-            platform="linux/$arch/$variant"
-            ;;
-        *)
-            err_msg "Unsupported architecture; $parch"
-            exit 1
-            ;;
-    esac
     info_msg "Selected platform: $platform"
 
     if [ "$container_engine" = "docker" ] && ! docker buildx version &>/dev/null; then
@@ -73,16 +109,15 @@ container.build() {
         set -e
         pyenv.activate
 
-        # Check if it is a git repository
         if [ ! -d .git ]; then
-            die 1 "This is not Git repository"
+            die 1 "this is not a Git repository"
         fi
 
+        # TODO: get current branch
         if ! git remote get-url origin &>/dev/null; then
-            die 1 "There is no remote origin"
+            die 1 "there is no remote origin"
         fi
 
-        # This is a git repository
         git update-index -q --refresh
         python -m searx.version freeze
         eval "$(python -m searx.version)"
@@ -90,42 +125,30 @@ container.build() {
         info_msg "Set \$DOCKER_TAG: $DOCKER_TAG"
         info_msg "Set \$GIT_URL: $GIT_URL"
 
-        # change cmp to lockfile when available
-        timestamp_requirements_main=$(git log -1 --format='%ct' ./requirements.txt)
-        timestamp_requirements_server=$(git log -1 --format='%ct' ./requirements-server.txt)
-        if [[ "$timestamp_requirements_main" -ge "$timestamp_requirements_server" ]]; then
-            timestamp_venv="$timestamp_requirements_main"
-        else
-            timestamp_venv="$timestamp_requirements_server"
-        fi
-
         if [ "$container_engine" = "podman" ]; then
-            params_build_builder="build --format=oci --platform=$platform --layers --identity-label=false --timestamp=$timestamp_venv"
+            params_build_builder="build --format=oci --layers --platform=$platform --identity-label=false"
-            params_build="build --format=oci --platform=$platform --layers --identity-label=false"
+            params_build="build --format=oci --layers --platform=$platform --identity-label=false"
         else
             params_build_builder="build --platform=$platform"
             params_build=$params_build_builder
         fi
 
-        if [ "$GITHUB_ACTIONS" = "true" ]; then
+        local_tag_arch="localhost/searxng/searxng:$DOCKER_TAG-$arch$variant"
-            params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
+        params_build+=" --tag=$local_tag_arch"
-        else
+        if [ "$GITHUB_ACTIONS" != "true" ]; then
-            params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest"
+            params_build+=" --tag=localhost/searxng/searxng:latest"
-            params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG"
+            params_build+=" --tag=localhost/searxng/searxng:$DOCKER_TAG"
         fi
 
         # shellcheck disable=SC2086
         "$container_engine" $params_build_builder \
-            --build-arg="TIMESTAMP_VENV=$timestamp_venv" \
+            --tag="localhost/searxng/searxng:builder" \
-            --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \
             --file="./container/builder.dockerfile" \
             .
         build_msg CONTAINER "Image \"builder\" built"
 
         # shellcheck disable=SC2086
         "$container_engine" $params_build \
-            --build-arg="CONTAINER_IMAGE_ORGANIZATION=$CONTAINER_IMAGE_ORGANIZATION" \
-            --build-arg="CONTAINER_IMAGE_NAME=$CONTAINER_IMAGE_NAME" \
             --build-arg="CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
             --build-arg="VERSION=$DOCKER_TAG" \
             --build-arg="VCS_URL=$GIT_URL" \
@@ -135,12 +158,17 @@ container.build() {
         build_msg CONTAINER "Image built"
 
         if [ "$GITHUB_ACTIONS" = "true" ]; then
-            "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
+            required_commands podman
+
+            mkdir -p "$tmpdir"
+            rm -f "$tmpdir"/*.tar
+            image_archive="$tmpdir/image_$arch$variant.tar"
+
+            podman save --format=oci-archive --output="$image_archive" "$local_tag_arch"
 
             # Output to GHA
             cat <<EOF >>"$GITHUB_OUTPUT"
 docker_tag=$DOCKER_TAG
-git_url=$GIT_URL
 EOF
         fi
     )
@@ -148,50 +176,27 @@ EOF
 }
 
 container.test() {
-    local parch=${OVERRIDE_ARCH:-$(uname -m)}
-    local arch
-    local variant
-    local platform
-
-    if [ "$GITHUB_ACTIONS" != "true" ]; then
-        die 1 "This command is intended to be run in GitHub Actions"
-    fi
-
     required_commands podman
 
-    # Setup arch specific
+    local image="$1"
-    case $parch in
+
-        "X64" | "x86_64" | "amd64")
+    if [ -z "$image" ]; then
-            arch="amd64"
+        image=$(find "$tmpdir" -type f -name '*.tar' -print -quit)
-            variant=""
+        if [ -z "$image" ]; then
-            platform="linux/$arch"
+            die 1 "no archives found in $tmpdir"
-            ;;
+        fi
-        "ARM64" | "aarch64" | "arm64")
+    fi
-            arch="arm64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARMV7" | "armhf" | "armv7l" | "armv7")
-            arch="arm"
-            variant="v7"
-            platform="linux/$arch/$variant"
-            ;;
-        *)
-            err_msg "Unsupported architecture; $parch"
-            exit 1
-            ;;
-    esac
-    build_msg CONTAINER "Selected platform: $platform"
 
     (
         set -e
 
-        podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
+        if [ -f "$image" ]; then
+            image=$(podman load --input="$image" | sed -n 's/^Loaded image: *//p' | tail -n1)
+        fi
 
-        name="$CONTAINER_IMAGE_NAME-$(date +%N)"
+        name="searxng-$(head -c 32 /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 16)"
 
-        podman create --name="$name" --rm --timeout=60 --network="host" \
+        podman create --name="$name" --rm --timeout=60 --network="host" "$image" >/dev/null
-            "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" >/dev/null
 
         podman start "$name" >/dev/null
         podman logs -f "$name" &
@@ -207,81 +212,32 @@ container.test() {
 }
 
 container.push() {
-    # Architectures on manifest
-    local release_archs=("amd64" "arm64" "armv7")
-
-    local archs=()
-    local variants=()
-    local platforms=()
-
-    if [ "$GITHUB_ACTIONS" != "true" ]; then
-        die 1 "This command is intended to be run in GitHub Actions"
-    fi
-
     required_commands podman
 
-    for arch in "${release_archs[@]}"; do
+    local release_tags=("$DOCKER_TAG" "latest")
-        case $arch in
+    local release_registries=("ghcr.io" "docker.io")
-            "X64" | "x86_64" | "amd64")
+
-                archs+=("amd64")
+    if [ "$GITHUB_ACTIONS" != "true" ]; then
-                variants+=("")
+        die 1 "This command is intended to be run in Actions"
-                platforms+=("linux/${archs[-1]}")
+    fi
-                ;;
+
-            "ARM64" | "aarch64" | "arm64")
+    if ! podman manifest exists "localhost/searxng/searxng:${release_tags[0]}" ||
-                archs+=("arm64")
+        ! podman manifest exists "localhost/searxng/searxng:${release_tags[1]}"; then
-                variants+=("")
+        container.__import_oci
-                platforms+=("linux/${archs[-1]}")
+    fi
-                ;;
-            "ARMV7" | "armv7" | "armhf" | "arm")
-                archs+=("arm")
-                variants+=("v7")
-                platforms+=("linux/${archs[-1]}/${variants[-1]}")
-                ;;
-            *)
-                err_msg "Unsupported architecture; $arch"
-                exit 1
-                ;;
-        esac
-    done
 
     (
         set -e
 
-        # Pull archs
-        for i in "${!archs[@]}"; do
-            podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}"
-        done
-
-        # Manifest tags ("latest" should be the last manifest)
-        release_tags=("$DOCKER_TAG" "latest")
-
-        # Create manifests
-        for tag in "${release_tags[@]}"; do
-            if ! podman manifest exists "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"; then
-                podman manifest create "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"
-            fi
-
-            # Add archs to manifest
-            for i in "${!archs[@]}"; do
-                podman manifest add \
-                    "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \
-                    "containers-storage:ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}"
-            done
-        done
-
         podman image list
 
-        # Remote registries
-        release_registries=("ghcr.io" "docker.io")
-
-        # Push manifests
         for registry in "${release_registries[@]}"; do
             for tag in "${release_tags[@]}"; do
                 build_msg CONTAINER "Pushing manifest $tag to $registry"
 
-                podman manifest push \
+                podman manifest push --all \
-                    "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \
+                    "localhost/searxng/searxng:$tag" \
-                    "docker://$registry/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"
+                    "docker://$registry/${GITHUB_REPOSITORY_OWNER:-"searxng"}/searxng:$tag"
             done
         done
     )