diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 296bfa5bf..6231a88d5 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -41,21 +41,10 @@ jobs: - runner: ubuntu-26.04-arm arch: armv7 - permissions: - packages: write - outputs: docker_tag: ${{ steps.build.outputs.docker_tag }} - git_url: ${{ steps.build.outputs.git_url }} steps: - - name: Login to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: "ghcr.io" - username: "${{ github.repository_owner }}" - password: "${{ secrets.GITHUB_TOKEN }}" - - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -93,6 +82,14 @@ jobs: OVERRIDE_ARCH: "${{ matrix.arch }}" run: make container.build + - name: Upload container image + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: container-image-${{ matrix.arch }} + path: "/var/tmp/searxng-podman/*.tar" + if-no-files-found: error + retention-days: 1 + test: name: Test (${{ matrix.arch }}) runs-on: ${{ matrix.runner }} @@ -109,13 +106,6 @@ jobs: arch: armv7 steps: - - name: Login to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: "ghcr.io" - username: "${{ github.repository_owner }}" - password: "${{ secrets.GITHUB_TOKEN }}" - - name: Setup QEMU uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 @@ -125,10 +115,16 @@ jobs: ref: "${{ github.event.workflow_run.head_sha || github.sha }}" persist-credentials: "false" + - name: Download container image + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: container-image-${{ matrix.arch }} + path: "/var/tmp/searxng-podman" + merge-multiple: true + - name: Test env: OVERRIDE_ARCH: "${{ matrix.arch }}" - GIT_URL: "${{ needs.build.outputs.git_url }}" run: make container.test release: @@ -163,8 +159,14 @@ jobs: ref: "${{ github.event.workflow_run.head_sha || github.sha }}" persist-credentials: "false" + - name: Download container images + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: container-image-* + path: "/var/tmp/searxng-podman" + merge-multiple: true + - name: Release env: - GIT_URL: "${{ needs.build.outputs.git_url }}" DOCKER_TAG: "${{ needs.build.outputs.docker_tag }}" run: make container.push diff --git a/container/builder.dockerfile b/container/builder.dockerfile index 7b92869d5..38ec42f9a 100644 --- a/container/builder.dockerfile +++ b/container/builder.dockerfile @@ -3,21 +3,15 @@ FROM docker.io/searxng/base:searxng-builder AS builder COPY ./requirements.txt ./requirements-server.txt ./ ENV UV_NO_MANAGED_PYTHON="true" -ENV UV_NATIVE_TLS="true" - -ARG TIMESTAMP_VENV="0" RUN --mount=type=cache,id=uv,target=/root/.cache/uv set -eux -o pipefail; \ - export SOURCE_DATE_EPOCH="$TIMESTAMP_VENV"; \ uv venv; \ uv pip install --requirements ./requirements.txt --requirements ./requirements-server.txt; \ uv cache prune --ci; \ find ./.venv/lib/ -type f -exec strip --strip-unneeded {} + || true; \ find ./.venv/lib/ -type d -name "__pycache__" -exec rm -rf {} +; \ find ./.venv/lib/ -type f -name "*.pyc" -delete; \ - python -m compileall -q -f -j 0 --invalidation-mode=unchecked-hash ./.venv/lib/; \ - find ./.venv/lib/python*/site-packages/*.dist-info/ -type f -name "RECORD" -exec sort -t, -k1,1 -o {} {} \;; \ - find ./.venv/ -exec touch -h --date="@$TIMESTAMP_VENV" {} + + python -m compileall -q -f -j 0 --invalidation-mode=unchecked-hash ./.venv/lib/ COPY --exclude=./searx/version_frozen.py ./searx/ ./searx/ diff --git a/container/dist.dockerfile b/container/dist.dockerfile index 3d23ab7db..b5766428f 100644 --- a/container/dist.dockerfile +++ b/container/dist.dockerfile @@ -1,7 +1,4 @@ -ARG CONTAINER_IMAGE_ORGANIZATION="searxng" -ARG CONTAINER_IMAGE_NAME="searxng" - -FROM localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder AS builder +FROM localhost/searxng/searxng:builder AS builder FROM docker.io/searxng/base:searxng AS dist COPY --chown=977:977 --from=builder /usr/local/searxng/.venv/ ./.venv/ diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh index 229008b37..a4a6db8ba 100644 --- a/utils/lib_sxng_container.sh +++ b/utils/lib_sxng_container.sh @@ -1,25 +1,82 @@ #!/usr/bin/env bash # SPDX-License-Identifier: AGPL-3.0-or-later +tmpdir="/var/tmp/searxng-podman/" + container.help() { cat </dev/null; then @@ -33,34 +90,13 @@ container.build() { elif command -v docker &>/dev/null; then container_engine="docker" else - die 42 "no compatible container engine is installed (podman or docker)" + die 42 "no compatible container engine is installed" fi fi info_msg "Selected engine: $container_engine" "$container_engine" version - # Setup arch specific - case $parch in - "X64" | "x86_64" | "amd64") - arch="amd64" - variant="" - platform="linux/$arch" - ;; - "ARM64" | "aarch64" | "arm64") - arch="arm64" - variant="" - platform="linux/$arch" - ;; - "ARMV7" | "armhf" | "armv7l" | "armv7") - arch="arm" - variant="v7" - platform="linux/$arch/$variant" - ;; - *) - err_msg "Unsupported architecture; $parch" - exit 1 - ;; - esac + container.__get_platform "${OVERRIDE_ARCH:-$(uname -m)}" info_msg "Selected platform: $platform" if [ "$container_engine" = "docker" ] && ! docker buildx version &>/dev/null; then @@ -73,16 +109,15 @@ container.build() { set -e pyenv.activate - # Check if it is a git repository if [ ! -d .git ]; then - die 1 "This is not Git repository" + die 1 "this is not a Git repository" fi + # TODO: get current branch if ! git remote get-url origin &>/dev/null; then - die 1 "There is no remote origin" + die 1 "there is no remote origin" fi - # This is a git repository git update-index -q --refresh python -m searx.version freeze eval "$(python -m searx.version)" @@ -90,42 +125,30 @@ container.build() { info_msg "Set \$DOCKER_TAG: $DOCKER_TAG" info_msg "Set \$GIT_URL: $GIT_URL" - # change cmp to lockfile when available - timestamp_requirements_main=$(git log -1 --format='%ct' ./requirements.txt) - timestamp_requirements_server=$(git log -1 --format='%ct' ./requirements-server.txt) - if [[ "$timestamp_requirements_main" -ge "$timestamp_requirements_server" ]]; then - timestamp_venv="$timestamp_requirements_main" - else - timestamp_venv="$timestamp_requirements_server" - fi - if [ "$container_engine" = "podman" ]; then - params_build_builder="build --format=oci --platform=$platform --layers --identity-label=false --timestamp=$timestamp_venv" - params_build="build --format=oci --platform=$platform --layers --identity-label=false" + params_build_builder="build --format=oci --layers --platform=$platform --identity-label=false" + params_build="build --format=oci --layers --platform=$platform --identity-label=false" else params_build_builder="build --platform=$platform" params_build=$params_build_builder fi - if [ "$GITHUB_ACTIONS" = "true" ]; then - params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" - else - params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest" - params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG" + local_tag_arch="localhost/searxng/searxng:$DOCKER_TAG-$arch$variant" + params_build+=" --tag=$local_tag_arch" + if [ "$GITHUB_ACTIONS" != "true" ]; then + params_build+=" --tag=localhost/searxng/searxng:latest" + params_build+=" --tag=localhost/searxng/searxng:$DOCKER_TAG" fi # shellcheck disable=SC2086 "$container_engine" $params_build_builder \ - --build-arg="TIMESTAMP_VENV=$timestamp_venv" \ - --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \ + --tag="localhost/searxng/searxng:builder" \ --file="./container/builder.dockerfile" \ . build_msg CONTAINER "Image \"builder\" built" # shellcheck disable=SC2086 "$container_engine" $params_build \ - --build-arg="CONTAINER_IMAGE_ORGANIZATION=$CONTAINER_IMAGE_ORGANIZATION" \ - --build-arg="CONTAINER_IMAGE_NAME=$CONTAINER_IMAGE_NAME" \ --build-arg="CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --build-arg="VERSION=$DOCKER_TAG" \ --build-arg="VCS_URL=$GIT_URL" \ @@ -135,12 +158,17 @@ container.build() { build_msg CONTAINER "Image built" if [ "$GITHUB_ACTIONS" = "true" ]; then - "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + required_commands podman + + mkdir -p "$tmpdir" + rm -f "$tmpdir"/*.tar + image_archive="$tmpdir/image_$arch$variant.tar" + + podman save --format=oci-archive --output="$image_archive" "$local_tag_arch" # Output to GHA cat <>"$GITHUB_OUTPUT" docker_tag=$DOCKER_TAG -git_url=$GIT_URL EOF fi ) @@ -148,50 +176,27 @@ EOF } container.test() { - local parch=${OVERRIDE_ARCH:-$(uname -m)} - local arch - local variant - local platform - - if [ "$GITHUB_ACTIONS" != "true" ]; then - die 1 "This command is intended to be run in GitHub Actions" - fi - required_commands podman - # Setup arch specific - case $parch in - "X64" | "x86_64" | "amd64") - arch="amd64" - variant="" - platform="linux/$arch" - ;; - "ARM64" | "aarch64" | "arm64") - arch="arm64" - variant="" - platform="linux/$arch" - ;; - "ARMV7" | "armhf" | "armv7l" | "armv7") - arch="arm" - variant="v7" - platform="linux/$arch/$variant" - ;; - *) - err_msg "Unsupported architecture; $parch" - exit 1 - ;; - esac - build_msg CONTAINER "Selected platform: $platform" + local image="$1" + + if [ -z "$image" ]; then + image=$(find "$tmpdir" -type f -name '*.tar' -print -quit) + if [ -z "$image" ]; then + die 1 "no archives found in $tmpdir" + fi + fi ( set -e - podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + if [ -f "$image" ]; then + image=$(podman load --input="$image" | sed -n 's/^Loaded image: *//p' | tail -n1) + fi - name="$CONTAINER_IMAGE_NAME-$(date +%N)" + name="searxng-$(head -c 32 /dev/urandom | tr -dc 'A-Za-z0-9' | head -c 16)" - podman create --name="$name" --rm --timeout=60 --network="host" \ - "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" >/dev/null + podman create --name="$name" --rm --timeout=60 --network="host" "$image" >/dev/null podman start "$name" >/dev/null podman logs -f "$name" & @@ -207,81 +212,32 @@ container.test() { } container.push() { - # Architectures on manifest - local release_archs=("amd64" "arm64" "armv7") - - local archs=() - local variants=() - local platforms=() - - if [ "$GITHUB_ACTIONS" != "true" ]; then - die 1 "This command is intended to be run in GitHub Actions" - fi - required_commands podman - for arch in "${release_archs[@]}"; do - case $arch in - "X64" | "x86_64" | "amd64") - archs+=("amd64") - variants+=("") - platforms+=("linux/${archs[-1]}") - ;; - "ARM64" | "aarch64" | "arm64") - archs+=("arm64") - variants+=("") - platforms+=("linux/${archs[-1]}") - ;; - "ARMV7" | "armv7" | "armhf" | "arm") - archs+=("arm") - variants+=("v7") - platforms+=("linux/${archs[-1]}/${variants[-1]}") - ;; - *) - err_msg "Unsupported architecture; $arch" - exit 1 - ;; - esac - done + local release_tags=("$DOCKER_TAG" "latest") + local release_registries=("ghcr.io" "docker.io") + + if [ "$GITHUB_ACTIONS" != "true" ]; then + die 1 "This command is intended to be run in Actions" + fi + + if ! podman manifest exists "localhost/searxng/searxng:${release_tags[0]}" || + ! podman manifest exists "localhost/searxng/searxng:${release_tags[1]}"; then + container.__import_oci + fi ( set -e - # Pull archs - for i in "${!archs[@]}"; do - podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" - done - - # Manifest tags ("latest" should be the last manifest) - release_tags=("$DOCKER_TAG" "latest") - - # Create manifests - for tag in "${release_tags[@]}"; do - if ! podman manifest exists "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"; then - podman manifest create "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" - fi - - # Add archs to manifest - for i in "${!archs[@]}"; do - podman manifest add \ - "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ - "containers-storage:ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" - done - done - podman image list - # Remote registries - release_registries=("ghcr.io" "docker.io") - - # Push manifests for registry in "${release_registries[@]}"; do for tag in "${release_tags[@]}"; do build_msg CONTAINER "Pushing manifest $tag to $registry" - podman manifest push \ - "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ - "docker://$registry/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + podman manifest push --all \ + "localhost/searxng/searxng:$tag" \ + "docker://$registry/${GITHUB_REPOSITORY_OWNER:-"searxng"}/searxng:$tag" done done )