[mod] container: rework builds (#6720)

Free from ghcr.io/searxng/cache! Now we save and load images without polluting
this shared (and useless) layer cache, which also makes it easy to download
artifacts for local debugging.

I also removed this nonsense about virtualenv normalization, this will be
properly fixed by using lockfiles on py dependencies.

utils/lib_sxng_container.sh

@@ -1,25 +1,82 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+tmpdir="/var/tmp/searxng-podman/"
+
 container.help() {
     cat <<EOF
 container.:
-  build     : build container image
+  build   : build container image
 EOF
 }
 
-CONTAINER_IMAGE_ORGANIZATION=${GITHUB_REPOSITORY_OWNER:-"searxng"}
-CONTAINER_IMAGE_NAME="searxng"
+container.__get_platform() {
+    local __arch="$1"
+
+    case $__arch in
+        "X64" | "x86_64" | "amd64")
+            arch="amd64"
+            variant=""
+            platform="linux/$arch"
+            ;;
+        "ARM64" | "aarch64" | "arm64")
+            arch="arm64"
+            variant=""
+            platform="linux/$arch"
+            ;;
+        "ARMV7" | "armhf" | "armv7l" | "armv7")
+            arch="arm"
+            variant="v7"
+            platform="linux/$arch/$variant"
+            ;;
+        *)
+            die 1 "unsupported architecture: $__arch"
+            ;;
+    esac
+}
+
+container.__import_oci() {
+    required_commands podman
+
+    local __release_tags=("$DOCKER_TAG" "latest")
+    local __archives=()
+
+    mkdir -p "$tmpdir"
+
+    while IFS= read -r -d '' archive; do
+        __archives+=("$archive")
+    done < <(find "$tmpdir" -maxdepth 1 -type f -name '*.tar' -print0)
+
+    if [ "${#__archives[@]}" -eq 0 ]; then
+        die 1 "no archives found in $tmpdir"
+    fi
+
+    (
+        set -e
+
+        podman manifest rm --ignore "${__release_tags[@]/#/localhost/searxng/searxng:}"
+
+        for tag in "${__release_tags[@]}"; do
+            podman manifest create "localhost/searxng/searxng:$tag"
+
+            for f in "${__archives[@]}"; do
+                podman manifest add "localhost/searxng/searxng:$tag" "oci-archive:$f"
+            done
+        done
+
+        podman manifest inspect "localhost/searxng/searxng:$DOCKER_TAG"
+    )
+    dump_return $?
+}
 
 container.build() {
-    local parch=${OVERRIDE_ARCH:-$(uname -m)}
+    required_commands git
+
     local container_engine
     local arch
     local variant
     local platform
 
-    required_commands git
-
     # Check if podman or docker is installed
     if [ "$1" = "podman" ] || [ "$1" = "docker" ]; then
         if ! command -v "$1" &>/dev/null; then
@@ -33,34 +90,13 @@ container.build() {
         elif command -v docker &>/dev/null; then
             container_engine="docker"
         else
-            die 42 "no compatible container engine is installed (podman or docker)"
+            die 42 "no compatible container engine is installed"
         fi
     fi
     info_msg "Selected engine: $container_engine"
     "$container_engine" version
 
-    # Setup arch specific
-    case $parch in
-        "X64" | "x86_64" | "amd64")
-            arch="amd64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARM64" | "aarch64" | "arm64")
-            arch="arm64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARMV7" | "armhf" | "armv7l" | "armv7")
-            arch="arm"
-            variant="v7"
-            platform="linux/$arch/$variant"
-            ;;
-        *)
-            err_msg "Unsupported architecture; $parch"
-            exit 1
-            ;;
-    esac
+    container.__get_platform "${OVERRIDE_ARCH:-$(uname -m)}"
     info_msg "Selected platform: $platform"
 
     if [ "$container_engine" = "docker" ] && ! docker buildx version &>/dev/null; then
@@ -73,16 +109,15 @@ container.build() {
         set -e
         pyenv.activate
 
-        # Check if it is a git repository
         if [ ! -d .git ]; then
-            die 1 "This is not Git repository"
+            die 1 "this is not a Git repository"
         fi
 
+        # TODO: get current branch
         if ! git remote get-url origin &>/dev/null; then
-            die 1 "There is no remote origin"
+            die 1 "there is no remote origin"
         fi
 
-        # This is a git repository
         git update-index -q --refresh
         python -m searx.version freeze
         eval "$(python -m searx.version)"
@@ -90,42 +125,30 @@ container.build() {
         info_msg "Set \$DOCKER_TAG: $DOCKER_TAG"
         info_msg "Set \$GIT_URL: $GIT_URL"
 
-        # change cmp to lockfile when available
-        timestamp_requirements_main=$(git log -1 --format='%ct' ./requirements.txt)
-        timestamp_requirements_server=$(git log -1 --format='%ct' ./requirements-server.txt)
-        if [[ "$timestamp_requirements_main" -ge "$timestamp_requirements_server" ]]; then
-            timestamp_venv="$timestamp_requirements_main"
-        else
-            timestamp_venv="$timestamp_requirements_server"
-        fi
-
         if [ "$container_engine" = "podman" ]; then
-            params_build_builder="build --format=oci --platform=$platform --layers --identity-label=false --timestamp=$timestamp_venv"
-            params_build="build --format=oci --platform=$platform --layers --identity-label=false"
+            params_build_builder="build --format=oci --layers --platform=$platform --identity-label=false"
+            params_build="build --format=oci --layers --platform=$platform --identity-label=false"
         else
             params_build_builder="build --platform=$platform"
             params_build=$params_build_builder
         fi
 
-        if [ "$GITHUB_ACTIONS" = "true" ]; then
-            params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
-        else
-            params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest"
-            params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG"
+        local_tag_arch="localhost/searxng/searxng:$DOCKER_TAG-$arch$variant"
+        params_build+=" --tag=$local_tag_arch"
+        if [ "$GITHUB_ACTIONS" != "true" ]; then
+            params_build+=" --tag=localhost/searxng/searxng:latest"
+            params_build+=" --tag=localhost/searxng/searxng:$DOCKER_TAG"
         fi
 
         # shellcheck disable=SC2086
         "$container_engine" $params_build_builder \
-            --build-arg="TIMESTAMP_VENV=$timestamp_venv" \
-            --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \
+            --tag="localhost/searxng/searxng:builder" \
             --file="./container/builder.dockerfile" \
             .
         build_msg CONTAINER "Image \"builder\" built"
 
         # shellcheck disable=SC2086
         "$container_engine" $params_build \
-            --build-arg="CONTAINER_IMAGE_ORGANIZATION=$CONTAINER_IMAGE_ORGANIZATION" \
-            --build-arg="CONTAINER_IMAGE_NAME=$CONTAINER_IMAGE_NAME" \
             --build-arg="CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
             --build-arg="VERSION=$DOCKER_TAG" \
             --build-arg="VCS_URL=$GIT_URL" \
@@ -135,12 +158,17 @@ container.build() {
         build_msg CONTAINER "Image built"
 
         if [ "$GITHUB_ACTIONS" = "true" ]; then
-            "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
+            required_commands podman
+
+            mkdir -p "$tmpdir"
+            rm -f "$tmpdir"/*.tar
+            image_archive="$tmpdir/image_$arch$variant.tar"
+
+            podman save --format=oci-archive --output="$image_archive" "$local_tag_arch"
 
             # Output to GHA
             cat <<EOF >>"$GITHUB_OUTPUT"
 docker_tag=$DOCKER_TAG
-git_url=$GIT_URL
 EOF
         fi
     )
@@ -148,50 +176,27 @@ EOF
 }
 
 container.test() {
-    local parch=${OVERRIDE_ARCH:-$(uname -m)}
-    local arch
-    local variant
-    local platform
-
-    if [ "$GITHUB_ACTIONS" != "true" ]; then
-        die 1 "This command is intended to be run in GitHub Actions"
-    fi
-
     required_commands podman
 
-    # Setup arch specific
-    case $parch in
-        "X64" | "x86_64" | "amd64")
-            arch="amd64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARM64" | "aarch64" | "arm64")
-            arch="arm64"
-            variant=""
-            platform="linux/$arch"
-            ;;
-        "ARMV7" | "armhf" | "armv7l" | "armv7")
-            arch="arm"
-            variant="v7"
-            platform="linux/$arch/$variant"
-            ;;
-        *)
-            err_msg "Unsupported architecture; $parch"
-            exit 1
-            ;;
-    esac
-    build_msg CONTAINER "Selected platform: $platform"
+    local image="$1"
+
+    if [ -z "$image" ]; then
+        image=$(find "$tmpdir" -type f -name '*.tar' -print -quit)
+        if [ -z "$image" ]; then
+            die 1 "no archives found in $tmpdir"
+        fi
+    fi
 
     (
         set -e
 
-        podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
+        if [ -f "$image" ]; then
+            image=$(podman load --input="$image" | sed -n 's/^Loaded image: *//p' | tail -n1)
+        fi
 
-        name="$CONTAINER_IMAGE_NAME-$(date +%N)"
+        name="searxng-$(head -c 32 /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 16)"
 
-        podman create --name="$name" --rm --timeout=60 --network="host" \
-            "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" >/dev/null
+        podman create --name="$name" --rm --timeout=60 --network="host" "$image" >/dev/null
 
         podman start "$name" >/dev/null
         podman logs -f "$name" &
@@ -207,81 +212,32 @@ container.test() {
 }
 
 container.push() {
-    # Architectures on manifest
-    local release_archs=("amd64" "arm64" "armv7")
-
-    local archs=()
-    local variants=()
-    local platforms=()
-
-    if [ "$GITHUB_ACTIONS" != "true" ]; then
-        die 1 "This command is intended to be run in GitHub Actions"
-    fi
-
     required_commands podman
 
-    for arch in "${release_archs[@]}"; do
-        case $arch in
-            "X64" | "x86_64" | "amd64")
-                archs+=("amd64")
-                variants+=("")
-                platforms+=("linux/${archs[-1]}")
-                ;;
-            "ARM64" | "aarch64" | "arm64")
-                archs+=("arm64")
-                variants+=("")
-                platforms+=("linux/${archs[-1]}")
-                ;;
-            "ARMV7" | "armv7" | "armhf" | "arm")
-                archs+=("arm")
-                variants+=("v7")
-                platforms+=("linux/${archs[-1]}/${variants[-1]}")
-                ;;
-            *)
-                err_msg "Unsupported architecture; $arch"
-                exit 1
-                ;;
-        esac
-    done
+    local release_tags=("$DOCKER_TAG" "latest")
+    local release_registries=("ghcr.io" "docker.io")
+
+    if [ "$GITHUB_ACTIONS" != "true" ]; then
+        die 1 "This command is intended to be run in Actions"
+    fi
+
+    if ! podman manifest exists "localhost/searxng/searxng:${release_tags[0]}" ||
+        ! podman manifest exists "localhost/searxng/searxng:${release_tags[1]}"; then
+        container.__import_oci
+    fi
 
     (
         set -e
 
-        # Pull archs
-        for i in "${!archs[@]}"; do
-            podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}"
-        done
-
-        # Manifest tags ("latest" should be the last manifest)
-        release_tags=("$DOCKER_TAG" "latest")
-
-        # Create manifests
-        for tag in "${release_tags[@]}"; do
-            if ! podman manifest exists "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"; then
-                podman manifest create "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"
-            fi
-
-            # Add archs to manifest
-            for i in "${!archs[@]}"; do
-                podman manifest add \
-                    "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \
-                    "containers-storage:ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}"
-            done
-        done
-
         podman image list
 
-        # Remote registries
-        release_registries=("ghcr.io" "docker.io")
-
-        # Push manifests
         for registry in "${release_registries[@]}"; do
             for tag in "${release_tags[@]}"; do
                 build_msg CONTAINER "Pushing manifest $tag to $registry"
 
-                podman manifest push \
-                    "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \
-                    "docker://$registry/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"
+                podman manifest push --all \
+                    "localhost/searxng/searxng:$tag" \
+                    "docker://$registry/${GITHUB_REPOSITORY_OWNER:-"searxng"}/searxng:$tag"
             done
         done
     )