Merge pull request #1629 from dalf/docker

Update Docker image

.dockerignore

@@ -0,0 +1,41 @@
+*~
+*/*~
+*/*/*~
+*/*/*/*~
+*/*/*/*/*~
+
+# Git
+.git
+.gitignore
+
+# CI
+.codeclimate.yml
+.travis.yml
+.taskcluster.yml
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*/__pycache__/
+*/*/__pycache__/
+*/*/*/__pycache__/
+*.py[cod]
+*/*.py[cod]
+*/*/*.py[cod]
+*/*/*/*.py[cod]
+
+# to sync with .gitignore
+.coverage
+coverage/
+.installed.cfg
+engines.cfg
+env
+searx-ve
+robot_log.html
+robot_output.xml
+robot_report.html
+test_basic/
+setup.cfg
+
+node_modules/
+
+.tx/

.gitignore

@@ -1,3 +1,4 @@
+# to sync with .dockerignore
 .coverage
 coverage/
 .installed.cfg

.travis.yml

@@ -1,19 +1,14 @@
+language: python
 sudo: false
 cache:
   - pip
   - npm
   - directories:
     - $HOME/.cache/pip
+
 addons:
   firefox: "latest"
-language: python
+
-python:
-  - "2.7"
-  - "3.5"
-  - "3.6"
-before_install:
-  - "export DISPLAY=:99.0"
-  - "sh -e /etc/init.d/xvfb start"
 install:
   - ./manage.sh install_geckodriver ~/drivers
   - export PATH=~/drivers:$PATH
@@ -27,6 +22,31 @@ script:
 after_success:
   - ./manage.sh py_test_coverage
   - codecov
+
+stages:
+  - test
+  - name: docker
+    if: branch = master AND type != pull_request AND env(DOCKER_USERNAME) IS present
+
+jobs:
+  include:
+    - python: "2.7"
+    - python: "3.5"
+    - python: "3.6"
+    - stage: docker
+      python: "3.6"
+      git:
+        depth: false
+      services:
+        - docker
+      addons: []
+      before_install: true
+      install: true
+      script:
+        - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        - ./manage.sh docker_build push
+      after_success: true
+
 notifications:
   irc:
     channels:

Dockerfile

@@ -1,57 +1,77 @@
-FROM alpine:3.8
+FROM alpine:3.10
-LABEL maintainer="searx <https://github.com/asciimoo/searx>"
+
-LABEL description="A privacy-respecting, hackable metasearch engine."
+ARG VERSION_GITCOMMIT=unknow
+ARG SEARX_GIT_VERSION=unknow
+
+ARG SEARX_GID=1000
+ARG SEARX_UID=1000
+
+ARG TIMESTAMP_SETTINGS=0
+ARG TIMESTAMP_UWSGI=0
+ARG LABEL_VCS_REF=
+ARG LABEL_VCS_URL=
+
+ENV BASE_URL= \
+    MORTY_KEY= \
+    MORTY_URL=
+EXPOSE 8080
+VOLUME /etc/searx
+VOLUME /var/log/uwsgi
 
-ENV BASE_URL=False IMAGE_PROXY=False HTTP_PROXY_URL= HTTPS_PROXY_URL=
-EXPOSE 8888
 WORKDIR /usr/local/searx
-CMD ["/sbin/tini","--","/usr/local/searx/run.sh"]
 
-RUN adduser -D -h /usr/local/searx -s /bin/sh searx searx \
+RUN addgroup -g ${SEARX_GID} searx && \
- && echo '#!/bin/sh' >> run.sh \
+    adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx
- && echo 'sed -i "s|base_url : False|base_url : $BASE_URL|g" searx/settings.yml' >> run.sh \
- && echo 'sed -i "s/image_proxy : False/image_proxy : $IMAGE_PROXY/g" searx/settings.yml' >> run.sh \
- && echo 'sed -i "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml' >> run.sh \
- && echo 'if [ -n "$HTTP_PROXY_URL" ] || [ -n "$HTTPS_PROXY_URL" ]; then' >> run.sh \
- && echo '  sed -i "s~^#    proxies :~    proxies:\\n      http: ${HTTP_PROXY_URL}\\n      https: ${HTTPS_PROXY_URL}\\n~" searx/settings.yml' >> run.sh \
- && echo 'fi' >> run.sh \
- && echo 'python searx/webapp.py' >> run.sh \
- && chmod +x run.sh
 
 COPY requirements.txt ./requirements.txt
 
-RUN echo "@commuedge http://nl.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
+RUN apk -U upgrade \
- && apk -U add \
+ && apk add -t build-dependencies \
     build-base \
-    python \
+    py3-setuptools \
-    python-dev \
+    python3-dev \
-    py-pip \
+    libffi-dev \
+    libxslt-dev \
+    libxml2-dev \
+    openssl-dev \
+    tar \
+    git \
+ && apk add \
+    ca-certificates \
+    su-exec \
+    python3 \
     libxml2 \
-    libxml2-dev \
     libxslt \
-    libxslt-dev \
-    libffi-dev \
     openssl \
-    openssl-dev \
+    tini \
-    ca-certificates \
+    uwsgi \
-    tini@commuedge \
+    uwsgi-python3 \
- && pip install --upgrade pip \
+ && pip3 install --upgrade pip \
- && pip install --no-cache -r requirements.txt \
+ && pip3 install --no-cache -r requirements.txt \
- && apk del \
+ && apk del build-dependencies \
-    build-base \
-    python-dev \
-    libffi-dev \
-    openssl-dev \
-    libxslt-dev \
-    libxml2-dev \
-    openssl-dev \
-    ca-certificates \
  && rm -f /var/cache/apk/*
 
-COPY . .
+COPY --chown=searx:searx . .
 
-RUN chown -R searx:searx *
+RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
+    touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \
+    touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \
+    if [ ! -z $VERSION_GITCOMMIT ]; then\
+      echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \
+    fi
 
-USER searx
+ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]
 
-RUN sed -i "s/127.0.0.1/0.0.0.0/g" searx/settings.yml
+# Keep this argument at the end since it change each time
+ARG LABEL_DATE=
+LABEL maintainer="searx <https://github.com/asciimoo/searx>" \
+      description="A privacy-respecting, hackable metasearch engine." \
+      version="${SEARX_GIT_VERSION}" \
+      org.label-schema.schema-version="1.0" \
+      org.label-schema.name="searx" \
+      org.label-schema.schema-version="${SEARX_GIT_VERSION}" \
+      org.label-schema.url="${LABEL_VCS_URL}" \
+      org.label-schema.vcs-ref=${LABEL_VCS_REF} \
+      org.label-schema.vcs-url=${LABEL_VCS_URL} \
+      org.label-schema.build-date="${LABEL_DATE}" \
+      org.label-schema.usage="https://github.com/searx/searx-docker"

dockerfiles/docker-entrypoint.sh

@@ -0,0 +1,128 @@
+#!/bin/sh
+
+export SEARX_VERSION=$(su searx -c 'python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)"')
+printf 'searx version %s\n\n' "${SEARX_VERSION}"
+
+export UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini
+export SEARX_SETTINGS_PATH=/etc/searx/settings.yml
+
+if [ -z "${BIND_ADDRESS}" ]; then
+    export BIND_ADDRESS=":8080"
+fi
+
+# Parse command line
+FORCE_CONF_UPDATE=0
+DRY_RUN=0
+while getopts "fdh" option
+do
+    case $option in
+	f)
+	    FORCE_CONF_UPDATE=1
+	    ;;
+	d)
+	    DRY_RUN=1
+	    ;;
+	h)
+	    printf "Command line:\n\n"
+	    printf "  -h  Display this help\n"
+	    printf "  -d  Dry run to update the configuration files.\n"
+	    printf "  -f  Always update on the configuration files (existing files are renamed with the .old suffix)\n"
+	    printf "      Without this option, new configuration files are copied with the .new suffix\n"
+	    printf "\nEnvironment variables:\n\n"
+	    printf "  BASE_URL      settings.yml : server.base_url\n"
+	    printf "  MORTY_URL     settings.yml : result_proxy.url\n"
+	    printf "  MORTY_KEY     settings.yml : result_proxy.key\n"
+	    printf "  BIND_ADDRESS  where uwsgi will accept HTTP request (format : host:port)\n"
+	    exit 0
+    esac
+done
+
+# helpers to update the configuration files
+patch_uwsgi_settings() {
+    CONF="$1"
+
+    # Nothing
+}
+
+patch_searx_settings() {
+    CONF="$1"
+
+    # Make sure that there is trailing slash at the end of BASE_URL
+    # see http://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion
+    export BASE_URL="${BASE_URL%/}/"
+
+    # update settings.yml
+    sed -i -e "s|base_url : False|base_url : ${BASE_URL}|g" \
+       -e "s/ultrasecretkey/$(openssl rand -hex 32)/g" \
+       "${CONF}"
+
+    # Morty configuration
+    if [ ! -z "${MORTY_KEY}" -a ! -z "${MORTY_URL}" ]; then
+	sed -i -e "s/image_proxy : False/image_proxy : True/g" \
+	    "${CONF}"
+	cat >> "${CONF}" <<-EOF
+
+# Morty configuration
+result_proxy:
+   url : ${MORTY_URL}
+   key : !!binary "${MORTY_KEY}"
+EOF
+    fi
+}
+
+update_conf() {
+    FORCE_CONF_UPDATE="$1"
+    CONF="$2"
+    NEW_CONF="${2}.new"
+    OLD_CONF="${2}.old"
+    REF_CONF="$3"
+    PATCH_REF_CONF="$4"
+
+    if [ -f "${CONF}" ]; then
+	if [ "${REF_CONF}" -nt "${CONF}" ]; then
+	    # There is a new version
+	    if [ $FORCE_CONF_UPDATE ]; then
+		# Replace the current configuration
+		printf '⚠️  Automaticaly update %s to the new version\n' "${CONF}"
+		if [ ! -f "${OLD_CONF}" ]; then
+		    printf 'The previous configuration is saved to %s\n' "${OLD_CONF}"
+		    mv "${CONF}" "${OLD_CONF}"
+		fi
+		cp "${REF_CONF}" "${CONF}"
+		$PATCH_REF_CONF "${CONF}"
+	    else
+		# Keep the current configuration
+		printf '⚠️  Check new version %s to make sure searx is working properly\n' "${NEW_CONF}"
+		cp "${REF_CONF}" "${NEW_CONF}"
+		$PATCH_REF_CONF "${NEW_CONF}"
+	    fi
+	else
+	    printf 'Use existing %s\n' "${CONF}"
+	fi
+    else
+	printf 'Create %s\n' "${CONF}"
+	cp "${REF_CONF}" "${CONF}"
+	$PATCH_REF_CONF "${CONF}"
+    fi
+}
+
+# make sure there are uwsgi settings
+update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searx/dockerfiles/uwsgi.ini" "patch_uwsgi_settings"
+
+# make sure there are searx settings
+update_conf "${FORCE_CONF_UPDATE}" "${SEARX_SETTINGS_PATH}" "/usr/local/searx/searx/settings.yml" "patch_searx_settings"
+
+# dry run (to update configuration files, then inspect them)
+if [ $DRY_RUN -eq 1 ]; then
+    printf 'Dry run\n'
+    exit
+fi
+
+#
+touch /var/run/uwsgi-logrotate
+chown -R searx:searx /var/log/uwsgi /var/run/uwsgi-logrotate
+unset MORTY_KEY
+
+# Start uwsgi
+printf 'Listen on %s\n' "${BIND_ADDRESS}"
+exec su-exec searx:searx uwsgi --master --http-socket "${BIND_ADDRESS}" "${UWSGI_SETTINGS_PATH}"

dockerfiles/uwsgi.ini

@@ -0,0 +1,33 @@
+[uwsgi]
+# Who will run the code
+uid = searx
+gid = searx
+
+# Number of workers (usually CPU count)
+workers = 4
+
+# The right granted on the created socket
+chmod-socket = 666
+
+# Plugin to use and interpretor config
+single-interpreter = true
+master = true
+plugin = python3
+lazy-apps = true
+enable-threads = true
+
+# Module to import
+module = searx.webapp
+
+# Virtualenv and python path
+pythonpath = /usr/local/searx/
+chdir = /usr/local/searx/searx/
+
+# Disable logging for privacy
+disable-logging=True
+
+# But keep errors for 2 days
+touch-logrotate = /run/uwsgi-logrotate
+unique-cron = 15 0 -1 -1 -1 { touch /run/uwsgi-logrotate  }
+log-backupname = /var/log/uwsgi/uwsgi.log.1
+logto = /var/log/uwsgi/uwsgi.log

manage.sh

@@ -158,6 +158,74 @@ grunt_build() {
     grunt --gruntfile "$SEARX_DIR/static/themes/simple/gruntfile.js"
 }
 
+docker_build() {
+    # Check if it is a git repository
+    if [ ! -d .git ]; then
+	echo "This is not Git repository"
+	exit 1
+    fi
+
+    if [ ! -x "$(which git)" ]; then
+	echo "git is not installed"
+	exit 1
+    fi
+
+    if [ ! git remote get-url origin 2> /dev/null ]; then
+	echo "there is no remote origin"
+	exit 1
+    fi
+
+    # This is a git repository
+
+    # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
+    # awk to remove the "v" and the "g"
+    SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); $3=substr($3, 2); print}')
+
+    # add the suffix "-dirty" if the repository has uncommited change
+    git update-index -q --refresh
+    if [ ! -z "$(git diff-index --name-only HEAD --)" ]; then
+	SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
+    fi
+
+    # Get the last git commit id, will be added to the Searx version (see Dockerfile)
+    VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4)
+    echo "Last commit : $VERSION_GITCOMMIT"
+
+    # Check consistency between the git tag and the searx/version.py file
+    # /!\ HACK : parse Python file with bash /!\
+    # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py )
+    # SEARX_PYTHON_VERSION=$(python -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
+    SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
+    if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then
+	echo "Inconsistency between the last git tag and the searx/version.py file"
+	echo "git tag:          $SEARX_GIT_VERSION"
+	echo "searx/version.py: $SEARX_PYTHON_VERSION"
+	exit 1
+    fi
+
+    # define the docker image name
+    # /!\ HACK to get the user name /!\
+    GITHUB_USER=$(git remote get-url origin | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
+    SEARX_IMAGE_NAME="${GITHUB_USER:-searx}/searx"
+
+    # build Docker image
+    echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
+    sudo docker build \
+         --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
+         --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
+         --build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
+         --build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \
+         --build-arg LABEL_VCS_URL=$(git remote get-url origin) \
+	 --build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \
+	 --build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \
+         -t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} .
+
+    if [ "$1" = "push" ]; then
+	sudo docker push ${SEARX_IMAGE_NAME}:latest
+	sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}
+    fi
+}
+
 #
 # Help
 #
@@ -182,6 +250,7 @@ Commands
     locales              - Compile locales
     styles               - Build less files
     grunt_build          - Build files for themes
+    docker_build         - Build Docker image
 
     Tests
     -----

searx/engines/dailymotion.py

@@ -26,7 +26,7 @@ language_support = True
 # see http://www.dailymotion.com/doc/api/obj-video.html
 search_url = 'https://api.dailymotion.com/videos?fields=created_time,title,description,duration,url,thumbnail_360_url,id&sort=relevance&limit=5&page={pageno}&{query}'  # noqa
 embedded_url = '<iframe frameborder="0" width="540" height="304" ' +\
-    'data-src="//www.dailymotion.com/embed/video/{videoid}" allowfullscreen></iframe>'
+    'data-src="https://www.dailymotion.com/embed/video/{videoid}" allowfullscreen></iframe>'
 
 supported_languages_url = 'https://api.dailymotion.com/languages'
 

searx/engines/vimeo.py

@@ -24,7 +24,7 @@ paging = True
 base_url = 'https://vimeo.com/'
 search_url = base_url + '/search/page:{pageno}?{query}'
 
-embedded_url = '<iframe data-src="//player.vimeo.com/video/{videoid}" ' +\
+embedded_url = '<iframe data-src="https://player.vimeo.com/video/{videoid}" ' +\
     'width="540" height="304" frameborder="0" ' +\
     'webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>'
 

searx/engines/youtube_api.py

@@ -23,7 +23,7 @@ base_url = 'https://www.googleapis.com/youtube/v3/search'
 search_url = base_url + '?part=snippet&{query}&maxResults=20&key={api_key}'
 
 embedded_url = '<iframe width="540" height="304" ' +\
-    'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\
+    'data-src="https://www.youtube-nocookie.com/embed/{videoid}" ' +\
     'frameborder="0" allowfullscreen></iframe>'
 
 base_youtube_url = 'https://www.youtube.com/watch?v='

searx/engines/youtube_noapi.py

@@ -30,7 +30,7 @@ time_range_dict = {'day': 'Ag',
                    'year': 'BQ'}
 
 embedded_url = '<iframe width="540" height="304" ' +\
-    'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\
+    'data-src="https://www.youtube-nocookie.com/embed/{videoid}" ' +\
     'frameborder="0" allowfullscreen></iframe>'
 
 base_youtube_url = 'https://www.youtube.com/watch?v='

searx/testing.py

@@ -71,7 +71,7 @@ class SearxRobotLayer():
 def run_robot_tests(tests):
     print('Running {0} tests'.format(len(tests)))
     for test in tests:
-        with Browser() as browser:
+        with Browser('firefox', headless=True) as browser:
             test(browser)