[upd] update to gecko driver v36

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

.editorconfig

@@ -14,6 +14,14 @@ charset = utf-8
 # code formatter accepts length of 120, but editor should prefer 80
 max_line_length = 80
 
+[{*.sh,manage}]
+indent_style = space
+indent_size = 4
+
+# shfmt options
+shell_variant  = bash
+switch_case_indent = true
+
 [*.html]
 # in the jinja templates we use indent size of 2 and we do not use tabs
 indent_size = 2

.github/dependabot.yml

@@ -33,6 +33,38 @@ updates:
           - "minor"
           - "patch"
 
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "friday"
+    open-pull-requests-limit: 5
+    target-branch: "master"
+    commit-message:
+      prefix: "[upd] searxng.org/devtools (Node.js):"
+    groups:
+      minor:
+        applies-to: version-updates
+        update-types:
+          - "minor"
+          - "patch"
+
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "friday"
+    open-pull-requests-limit: 5
+    target-branch: "master"
+    commit-message:
+      prefix: "[upd] searxng.org/devtools (Go):"
+    groups:
+      minor:
+        applies-to: version-updates
+        update-types:
+          - "minor"
+          - "patch"
+
   - package-ecosystem: "docker"
     directory: "/"
     schedule:

.gitignore

@@ -9,6 +9,7 @@ geckodriver.log
 .coverage
 coverage/
 
+.govm/
 .nvm/
 cache/
 build/

Makefile

@@ -32,16 +32,13 @@ install uninstall:
 	$(Q)./manage pyenv.$@
 
 PHONY += clean
-clean: py.clean docs.clean node.clean nvm.clean test.clean
+clean: py.clean docs.clean node.clean nvm.clean go.clean test.clean
 	$(Q)./manage build_msg CLEAN  "common files"
 	$(Q)find . -name '*.orig' -exec rm -f {} +
 	$(Q)find . -name '*.rej' -exec rm -f {} +
 	$(Q)find . -name '*~' -exec rm -f {} +
 	$(Q)find . -name '*.bak' -exec rm -f {} +
 
-lxc.clean:
-	$(Q)rm -rf lxc-env
-
 PHONY += search.checker search.checker.%
 search.checker: install
 	$(Q)./manage pyenv.cmd searxng-checker -v
@@ -50,8 +47,8 @@ search.checker.%: install
 	$(Q)./manage pyenv.cmd searxng-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
 
 PHONY += test ci.test test.shell
-ci.test: test.yamllint test.black test.types.ci  test.pylint test.unit test.robot test.rst test.shell test.pybabel
+test:    test.yamllint test.black test.pyright test.pylint test.unit test.robot test.rst test.shell test.shfmt
-test:    test.yamllint test.black test.types.dev test.pylint test.unit test.robot test.rst test.shell
+ci.test: test test.pybabel
 test.shell:
 	$(Q)shellcheck -x -s dash \
 		container/entrypoint.sh
@@ -60,15 +57,15 @@ test.shell:
 		$(MTOOLS) \
 		utils/lib.sh \
 		utils/lib_sxng*.sh \
-		utils/lib_go.sh \
+		utils/lib_govm.sh \
 		utils/lib_nvm.sh \
 		utils/lib_redis.sh \
 		utils/lib_valkey.sh \
-		utils/searxng.sh \
+		utils/searxng.sh
-		utils/lxc.sh \
-		utils/lxc-searxng.env
 	$(Q)$(MTOOLS) build_msg TEST "$@ OK"
 
+PHONY += format
+format: format.python format.shell
 
 # wrap ./manage script
 
@@ -82,11 +79,12 @@ MANAGE += gecko.driver
 MANAGE += node.env node.env.dev node.clean
 MANAGE += py.build py.clean
 MANAGE += pyenv pyenv.install pyenv.uninstall
-MANAGE += format.python
+MANAGE += format.python format.shell
-MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.coverage test.robot test.rst test.clean test.themes test.types.dev test.types.ci
+MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.coverage test.robot test.rst test.clean test.themes test.pyright test.shfmt
 MANAGE += themes.all themes.simple themes.fix themes.lint themes.test
 MANAGE += static.build.commit static.build.drop static.build.restore
 MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
+MANAGE += go.env.dev go.clean
 
 PHONY += $(MANAGE)
 

docs/dev/index.rst

@@ -17,7 +17,6 @@ Developer documentation
    plugins/index
    answerers/index
    translation
-   lxcdev
    makefile
    reST
    searxng_extra/index

docs/dev/lxcdev.rst

@@ -1,437 +0,0 @@
-.. _lxcdev:
-
-==============================
-Developing in Linux Containers
-==============================
-
-.. _LXC: https://linuxcontainers.org/lxc/introduction/
-
-In this article we will show, how you can make use of Linux Containers (LXC_) in
-*distributed and heterogeneous development cycles* (TL;DR; jump to the
-:ref:`lxcdev summary`).
-
-.. sidebar:: Audience
-
-   This blog post is written for experienced admins and developers.  Readers
-   should have a serious meaning about the terms: *distributed*, *merge* and
-   *linux container*.
-
-   **hint**
-
-   If you have issues with the internet connectivity of your containers read
-   section :ref:`internet connectivity docker`.
-
-
-.. contents::
-   :depth: 2
-   :local:
-   :backlinks: entry
-
-
-Motivation
-==========
-
-Most often in our development cycle, we edit the sources and run some test
-and/or builds by using ``make`` :ref:`[ref] <makefile>` before we commit.  This
-cycle is simple and perfect but might fail in some aspects we should not
-overlook.
-
-  **The environment in which we run all our development processes matters!**
-
-The :ref:`makefile` and the :ref:`make install` encapsulate a lot for us, but
-these tools do not have access to all prerequisites.  For example, there may
-have dependencies on packages that are installed on developer's desktop, but
-usually are not preinstalled on a server or client system.  Another example is;
-settings have been made to the software on developer's desktop that would never
-be set on a *production* system.
-
-  **Linux Containers are isolate environments**, we use them to not mix up all
-  the prerequisites from various projects on developer's desktop.
-
-The scripts from :ref:`searx_utils` can divide in those to install and maintain
-software
-
-- :ref:`searxng.sh`
-
-and the script
-
-- :ref:`lxc.sh`
-
-with we can scale our installation, maintenance or even development tasks over a
-stack of isolated containers / what we call the:
-
-- :ref:`searxng lxc suite`
-
-.. _lxcdev install searxng:
-
-Gentlemen, start your engines!
-==============================
-
-.. _LXD: https://linuxcontainers.org/lxd/introduction/
-.. _archlinux: https://www.archlinux.org/
-
-Before you can start with containers, you need to install and initiate LXD_
-once:
-
-.. tabs::
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ snap install lxd
-        $ lxd init --auto
-
-And you need to clone from origin or if you have your own fork, clone from your
-fork:
-
-.. tabs::
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ cd ~/Downloads
-        $ git clone https://github.com/searxng/searxng.git searxng
-        $ cd searxng
-
-.. sidebar:: The ``searxng-archlinux`` container
-
-   is the base of all our exercises here.
-
-The :ref:`lxc-searxng.env` consists of several images, see ``export
-LXC_SUITE=(...`` near by :origin:`utils/lxc-searxng.env#L19`.
-For this blog post we exercise on a archlinux_ image.  The container of this
-image is named ``searxng-archlinux``.
-
-Lets build the container, but be sure that this container does not already
-exists, so first lets remove possible old one:
-
-.. tabs::
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh remove searxng-archlinux
-        $ sudo -H ./utils/lxc.sh build searxng-archlinux
-
-
-.. sidebar::  further read
-
-   - :ref:`lxc.sh install suite`
-   - :ref:`installation nginx`
-
-To install the complete :ref:`SearXNG suite <searxng lxc suite>` and the HTTP
-proxy :ref:`installation nginx` into the archlinux container run:
-
-.. tabs::
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh install suite searxng-archlinux
-        $ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx
-        $ sudo ./utils/lxc.sh show suite | grep SEARXNG_URL
-        ...
-        [searxng-archlinux]    SEARXNG_URL          : http://n.n.n.140/searxng
-
-.. sidebar:: Fully functional SearXNG suite
-
-   From here on you have a fully functional SearXNG suite (including a
-   :ref:`valkey db`).
-
-In such a SearXNG suite admins can maintain and access the debug log of the
-services quite easy.
-
-In the example above the SearXNG instance in the container is wrapped to
-``http://n.n.n.140/searxng`` to the HOST system.  Note, on your HOST system, the
-IP of your ``searxng-archlinux`` container is different to this example.  To
-test the instance in the container from outside of the container, in your WEB
-browser on your desktop just open the URL reported in your installation
-
-.. _working in containers:
-
-In containers, work as usual
-============================
-
-Usually you open a root-bash using ``sudo -H bash``.  In case of LXC containers
-open the root-bash in the container is done by the ``./utils/lxc.sh cmd
-searxng-archlinux`` command:
-
-.. tabs::
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash
-        INFO:  [searxng-archlinux] bash
-        [root@searxng-archlinux SearXNG]$
-
-The prompt ``[root@searxng-archlinux ...]`` signals, that you are the root user
-in the container (GUEST).  To debug the running SearXNG instance use:
-
-.. tabs::
-
-  .. group-tab:: ``[root@searxng-archlinux SearXNG]`` (GUEST)
-
-     .. code:: bash
-
-        $ ./utils/searxng.sh instance inspect
-        ...
-        use [CTRL-C] to stop monitoring the log
-        ...
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh instance inspect
-        ...
-        use [CTRL-C] to stop monitoring the log
-        ...
-
-
-Back in the browser on your desktop open the service http://n.n.n.140/searxng
-and run your application tests while the debug log is shown in the terminal from
-above.  You can stop monitoring using ``CTRL-C``, this also disables the *"debug
-option"* in SearXNG's settings file and restarts the SearXNG uwsgi application.
-
-Another point we have to notice is that the service :ref:`SearXNG <searxng.sh>`
-runs under dedicated system user account with the same name (compare
-:ref:`create searxng user`).  To get a login shell from these accounts, simply
-call:
-
-.. tabs::
-
-  .. group-tab:: ``[root@searxng-archlinux SearXNG]`` (GUEST)
-
-     .. code:: bash
-
-        $ ./utils/searxng.sh instance cmd bash -l
-        (searx-pyenv) [searxng@searxng-archlinux ~]$ pwd
-        /usr/local/searxng
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh instance cmd bash -l
-        INFO:  [searxng-archlinux] ./utils/searxng.sh instance cmd bash -l
-        (searx-pyenv) [searxng@searxng-archlinux ~]$ pwd
-        /usr/local/searxng
-
-The prompt ``[searxng@searxng-archlinux]`` signals that you are logged in as system
-user ``searxng`` in the ``searxng-archlinux`` container and the python *virtualenv*
-``(searxng-pyenv)`` environment is activated.
-
-
-Wrap production into developer suite
-====================================
-
-In this section we will see how to change the *"Fully functional SearXNG suite"*
-from a LXC container (which is quite ready for production) into a developer
-suite.  For this, we have to keep an eye on the :ref:`installation basic`:
-
-- SearXNG setup in: ``/etc/searxng/settings.yml``
-- SearXNG user's home: ``/usr/local/searxng``
-- virtualenv in: ``/usr/local/searxng/searxng-pyenv``
-- SearXNG software in: ``/usr/local/searxng/searxng-src``
-
-With the use of the :ref:`searxng.sh` the SearXNG service was installed as
-:ref:`uWSGI application <searxng uwsgi>`.  To maintain this service, we can use
-``systemctl`` (compare :ref:`uWSGI maintenance`).
-
-.. tabs::
-
-  .. group-tab:: uwsgi@searxng
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux systemctl stop uwsgi@searxng
-
-With the command above, we stopped the SearXNG uWSGI-App in the archlinux
-container.
-
-The uWSGI-App for the archlinux distros is configured in
-:origin:`utils/templates/etc/uwsgi/apps-archlinux/searxng.ini`, from where at
-least you should attend the settings of ``uid``, ``chdir``, ``env`` and
-``http``::
-
-  env = SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml
-  http = 127.0.0.1:8888
-
-  chdir = /usr/local/searxng/searxng-src/searx
-  virtualenv = /usr/local/searxng/searxng-pyenv
-  pythonpath = /usr/local/searxng/searxng-src
-
-If you have read the :ref:`Good to know` you remember, that each container
-shares the root folder of the repository and the command ``utils/lxc.sh cmd``
-handles relative path names **transparent**.
-
-To wrap the SearXNG installation in the container into a developer one, we
-simple have to create a symlink to the **transparent** repository from the
-desktop.  Now lets replace the repository at ``searxng-src`` in the container
-with the working tree from outside of the container:
-
-.. tabs::
-
-  .. group-tab:: ``[root@searxng-archlinux SearXNG]`` (GUEST)
-
-     .. code:: bash
-
-        $ mv /usr/local/searxng/searxng-src  /usr/local/searxng/searxng-src.old
-        $ ln -s /share/SearXNG/ /usr/local/searxng/searxng-src
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
-          mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \
-          ln -s /share/SearXNG/ /usr/local/searxng/searxng-src
-
-Now we can develop as usual in the working tree of our desktop system.  Every
-time the software was changed, you have to restart the SearXNG service (in the
-container):
-
-.. tabs::
-
-  .. group-tab:: uwsgi@searxng
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux systemctl restart uwsgi@searxng
-
-
-Remember: :ref:`working in containers` .. here are just some examples from my
-daily usage:
-
-To *inspect* the SearXNG instance (already described above):
-
-.. tabs::
-
-  .. group-tab:: ``[root@searxng-archlinux SearXNG]`` (GUEST)
-
-     .. code:: bash
-
-        $ ./utils/searxng.sh inspect service
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh inspect service
-
-Run :ref:`makefile`, e.g. to test inside the container:
-
-.. tabs::
-
-  .. group-tab:: ``[root@searxng-archlinux SearXNG]`` (GUEST)
-
-     .. code:: bash
-
-        $ make test
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux  make test
-
-
-
-To install all prerequisites needed for a :ref:`buildhosts`:
-
-.. tabs::
-
-  .. group-tab:: ``[root@searxng-archlinux SearXNG]`` (GUEST)
-
-     .. code:: bash
-
-        $ ./utils/searxng.sh install buildhost
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh install buildhost
-
-
-To build the docs on a buildhost :ref:`buildhosts`:
-
-.. tabs::
-
-  .. group-tab:: ``[root@searxng-archlinux SearXNG]`` (GUEST)
-
-     .. code:: bash
-
-        $ make docs.html
-
-  .. group-tab:: desktop (HOST)
-
-     .. code:: bash
-
-        $ sudo -H ./utils/lxc.sh cmd searxng-archlinux make docs.html
-
-
-.. _lxcdev summary:
-
-Summary
-=======
-
-We build up a fully functional SearXNG suite in a archlinux container:
-
-.. code:: bash
-
-   $ sudo -H ./utils/lxc.sh build searxng-archlinux
-   $ sudo -H ./utils/lxc.sh install suite searxng-archlinux
-   ...
-   Developer install? (wraps source from HOST into the running instance) [YES/no]
-
-To wrap the suite into a developer one answer ``YES`` (or press Enter).
-
-.. code:: text
-
-   link SearXNG's sources to: /share/SearXNG
-   =========================================
-
-   mv -f "/usr/local/searxng/searxng-src" "/usr/local/searxng/searxng-src.backup"
-   ln -s "/share/SearXNG" "/usr/local/searxng/searxng-src"
-   ls -ld /usr/local/searxng/searxng-src
-     |searxng| lrwxrwxrwx 1 searxng searxng ... /usr/local/searxng/searxng-src -> /share/SearXNG
-
-On code modification the instance has to be restarted (see :ref:`uWSGI
-maintenance`):
-
-.. code:: bash
-
-   $ sudo -H ./utils/lxc.sh cmd searxng-archlinux systemctl restart uwsgi@searxng
-
-To access HTTP from the desktop we installed nginx for the services inside the
-container:
-
-.. code:: bash
-
-   $ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx
-
-To get information about the SearxNG suite in the archlinux container we can
-use:
-
-.. code:: text
-
-   $ sudo -H ./utils/lxc.sh show suite searxng-archlinux
-   [searxng-archlinux]  INFO:  (eth0) docs-live:  http:///n.n.n.140:8080/
-   [searxng-archlinux]  INFO:  (eth0) IPv6:       http://[fd42:555b:2af9:e121:216:3eff:fe5b:1744]
-   [searxng-archlinux]  uWSGI:
-   [searxng-archlinux]    SEARXNG_UWSGI_SOCKET : /usr/local/searxng/run/socket
-   [searxng-archlinux]  environment /usr/local/searxng/searxng-src/utils/brand.env:
-   [searxng-archlinux]    GIT_URL              : https://github.com/searxng/searxng
-   [searxng-archlinux]    GIT_BRANCH           : master
-   [searxng-archlinux]    SEARXNG_URL          : http:///n.n.n.140/searxng
-   [searxng-archlinux]    SEARXNG_PORT         : 8888
-   [searxng-archlinux]    SEARXNG_BIND_ADDRESS : 127.0.0.1

docs/dev/makefile.rst

@@ -188,24 +188,22 @@ sources of the theme need to be rebuild.  You can do that by running::
 
     $ LIVE_THEME=simple make run
 
-.. _make format.python:
+.. _make format:
 
-``make format.python``
+``make format``
 ======================
 
-Format Python source code using `Black code style`_.  See ``$BLACK_OPTIONS``
-and ``$BLACK_TARGETS`` in :origin:`Makefile`.
-
-.. attention::
-
-   We stuck at Black 22.12.0, please read comment in PR `Bump black from 22.12.0
-   to 23.1.0`_
-
-.. _Bump black from 22.12.0 to 23.1.0:
-   https://github.com/searxng/searxng/pull/2159#pullrequestreview-1284094735
-
 .. _Black code style:
    https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html
+.. _shfmt: https://github.com/mvdan/sh?tab=readme-ov-file#shfmt
+.. _EditorConfig: https://github.com/patrickvane/shfmt?tab=readme-ov-file#description
+
+- Format Python source code using `Black code style`_.  See ``$BLACK_OPTIONS``
+  and ``$BLACK_TARGETS`` in :origin:`Makefile`.
+
+- Format Shell scripts using shfmt_. The formatter ``shfmt`` reads the rules
+  from the EditorConfig_ files.
+
 
 .. _make clean:
 

docs/dev/quickstart.rst

@@ -27,7 +27,7 @@ Here is how a minimal workflow looks like:
 
 1. *start* hacking
 2. *run* your code: :ref:`make run`
-3. *format & test* your code: :ref:`make format.python` and :ref:`make test`
+3. *format & test* your code: :ref:`make format` and :ref:`make test`
 
 If you think at some point something fails, go back to *start*.  Otherwise,
 choose a meaningful commit message and we are happy to receive your pull

docs/utils/index.rst

@@ -12,7 +12,7 @@ and developers.
    :maxdepth: 2
 
    searxng.sh
-   lxc.sh
+
 
 Common command environments
 ===========================
@@ -24,8 +24,7 @@ The scripts in our tooling box often dispose of common environments:
 ``FORCE_TIMEOUT`` : environment
   Sets timeout for interactive prompts. If you want to run a script in batch
   job, with defaults choices, set ``FORCE_TIMEOUT=0``.  By example; to install a
-  SearXNG server and nginx proxy on all containers of the :ref:`SearXNG suite
+  SearXNG server and nginx proxy use::
-  <lxc-searxng.env>` use::
 
-    sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install all
+    $ FORCE_TIMEOUT=0 ./utils/searxng.sh install all
-    sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx
+    $ FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx

docs/utils/lxc.sh.rst

@@ -1,295 +0,0 @@
-
-.. _snap: https://snapcraft.io
-.. _snapcraft LXD: https://snapcraft.io/lxd
-.. _LXC/LXD Image Server: https://uk.images.linuxcontainers.org/
-.. _LXC: https://linuxcontainers.org/lxc/introduction/
-.. _LXD: https://linuxcontainers.org/lxd/introduction/
-.. _`LXD@github`: https://github.com/lxc/lxd
-
-.. _archlinux: https://www.archlinux.org/
-
-.. _lxc.sh:
-
-================
-``utils/lxc.sh``
-================
-
-With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of
-containers, what we call the: *lxc suite*.  The :ref:`lxc-searxng.env` is
-loaded by default, every time you start the ``lxc.sh`` script (*you do not need
-to care about*).
-
-.. sidebar:: further reading
-
-   - snap_, `snapcraft LXD`_
-   - LXC_,  LXD_
-   - `LXC/LXD Image Server`_
-   - `LXD@github`_
-
-.. contents::
-   :depth: 2
-   :local:
-   :backlinks: entry
-
-
-.. _lxd install:
-
-Install LXD
-===========
-
-Before you can start with containers, you need to install and initiate LXD_
-once::
-
-  $ snap install lxd
-  $ lxd init --auto
-
-To make use of the containers from the *SearXNG suite*, you have to build the
-:ref:`LXC suite containers <lxc.sh help>` initial.  But be warned, **this might
-take some time**::
-
-  $ sudo -H ./utils/lxc.sh build
-
-.. sidebar:: hint
-
-   If you have issues with the internet connectivity of your containers read
-   section :ref:`internet connectivity docker`.
-
-A cup of coffee later, your LXC suite is build up and you can run whatever task
-you want / in a selected or even in all :ref:`LXC suite containers <lxc.sh
-help>`.
-
-.. _internet connectivity docker:
-
-Internet Connectivity & Docker
-------------------------------
-
-.. sidebar::  further read
-
-   - `Docker blocking network of existing LXC containers <https://github.com/docker/for-linux/issues/103>`__
-   - `Docker and IPtables (fralef.me) <https://fralef.me/docker-and-iptables.html>`__
-   - `Docker and iptables (docker.com) <https://docs.docker.com/network/iptables/#docker-on-a-router/>`__
-
-There is a conflict in the ``iptables`` setup of Docker & LXC.  If you have
-docker installed, you may find that the internet connectivity of your LXD
-containers no longer work.
-
-Whenever docker is started (reboot) it sets the iptables policy for the
-``FORWARD`` chain to ``DROP`` `[ref]
-<https://docs.docker.com/network/iptables/#docker-on-a-router>`__::
-
-  $ sudo -H iptables-save | grep FORWARD
-  :FORWARD ACCEPT [7048:7851230]
-  :FORWARD DROP [7048:7851230]
-
-A handy solution of this problem might be to reset the policy for the
-``FORWARD`` chain after the network has been initialized.  For this create a
-file in the ``if-up`` section of the network (``/etc/network/if-up.d/iptable``)
-and insert the following lines::
-
-  #!/bin/sh
-  iptables -F FORWARD
-  iptables -P FORWARD ACCEPT
-
-Don't forget to set the execution bit::
-
-  sudo chmod ugo+x /etc/network/if-up.d/iptable
-
-Reboot your system and check the iptables rules::
-
-  $ sudo -H iptables-save | grep FORWARD
-  :FORWARD ACCEPT [7048:7851230]
-  :FORWARD ACCEPT [7048:7851230]
-
-
-.. _searxng lxc suite:
-
-SearXNG LXC suite
-=================
-
-The intention of the *SearXNG LXC suite* is to build up a suite of containers
-for development tasks or :ref:`buildhosts <Setup SearXNG buildhost>` with a very
-small set of simple commands.  At the end of the ``--help`` output the SearXNG
-suite from the :ref:`lxc-searxng.env` is introduced::
-
-   $ sudo -H ./utils/lxc.sh --help
-   ...
-   LXC suite: searxng
-     Suite includes installation of SearXNG
-     images:     ubu2004 ubu2204 fedora35 archlinux
-     containers: searxng-ubu2004 searxng-ubu2204 searxng-fedora35 searxng-archlinux
-
-As shown above there are images and containers build up on this images.  To show
-more info about the containers in the *SearXNG LXC suite* call ``show suite``.
-If this is the first time you make use of the SearXNG LXC suite, no containers
-are installed and the output is::
-
-  $ sudo -H ./utils/lxc.sh show suite
-
-  LXC suite (searxng-*)
-  =====================
-
-  +------+-------+------+------+------+-----------+
-  | NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS |
-  +------+-------+------+------+------+-----------+
-
-  WARN:  container searxng-ubu2004 does not yet exists
-  WARN:  container searxng-ubu2204 does not yet exists
-  WARN:  container searxng-fedora35 does not yet exists
-  WARN:  container searxng-archlinux does not yet exists
-
-If you do not want to run a command or a build in all containers, **you can
-build just one**. Here by example in the container that is build upon the
-*archlinux* image::
-
-  $ sudo -H ./utils/lxc.sh build searxng-archlinux
-  $ sudo -H ./utils/lxc.sh cmd searxng-archlinux pwd
-
-Otherwise, to apply a command to all containers you can use::
-
-  $ sudo -H ./utils/lxc.sh build
-  $ sudo -H ./utils/lxc.sh cmd -- ls -la .
-
-Running commands
-----------------
-
-**Inside containers, you can run scripts** from the :ref:`toolboxing` or run
-what ever command you need.  By example, to start a bash use::
-
-  $ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash
-  INFO:  [searxng-archlinux] bash
-  [root@searxng-archlinux SearXNG]#
-
-.. _Good to know:
-
-Good to know
-------------
-
-Each container shares the root folder of the repository and the command
-``utils/lxc.sh cmd`` **handle relative path names transparent**::
-
- $ pwd
- /share/SearXNG
-
- $ sudo -H ./utils/lxc.sh cmd searxng-archlinux pwd
- INFO:  [searxng-archlinux] pwd
- /share/SearXNG
-
-The path ``/share/SearXNG`` will be different on your HOST system.  The commands
-in the container are executed by the ``root`` inside of the container.  Compare
-output of::
-
-  $ ls -li Makefile
-  47712402 -rw-rw-r-- 1 markus markus 2923 Apr 19 13:52 Makefile
-
-  $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ls -li Makefile
-  INFO:  [searxng-archlinux] ls -li Makefile
-  47712402 -rw-rw-r-- 1 root root 2923 Apr 19 11:52 Makefile
-  ...
-
-Since the path ``/share/SearXNG`` of the HOST system is wrapped into the
-container under the same name, the shown ``Makefile`` (inode ``47712402``) in
-the output is always the identical ``/share/SearXNG/Makefile`` from the HOST
-system.  In the example shown above the owner of the path in the container is
-the ``root`` user of the container (and the timezone in the container is
-different to HOST system).
-
-
-.. _lxc.sh install suite:
-
-Install suite
--------------
-
-.. sidebar::  further read
-
-   - :ref:`working in containers`
-   - :ref:`FORCE_TIMEOUT <FORCE_TIMEOUT>`
-
-To install the complete :ref:`SearXNG suite <lxc-searxng.env>` into **all** LXC_
-containers leave the container argument empty and run::
-
-  $ sudo -H ./utils/lxc.sh build
-  $ sudo -H ./utils/lxc.sh install suite
-
-To *build & install* suite only in one container you can use by example::
-
-  $ sudo -H ./utils/lxc.sh build searxng-archlinux
-  $ sudo -H ./utils/lxc.sh install suite searxng-archlinux
-
-The command above installs a SearXNG suite (see :ref:`installation scripts`).
-To :ref:`install a nginx <installation nginx>` reverse proxy (or alternatively
-use :ref:`apache <installation apache>`)::
-
-  $ sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx
-
-Same operation just in one container of the suite::
-
-  $ sudo -H ./utils/lxc.sh cmd searxng-archlinux FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx
-
-The :ref:`FORCE_TIMEOUT <FORCE_TIMEOUT>` environment is set to zero to run the
-script without user interaction.
-
-To get the IP (URL) of the SearXNG service in the containers use ``show suite``
-command.  To test instances from containers just open the URLs in your
-WEB-Browser::
-
-  $ sudo ./utils/lxc.sh show suite | grep SEARXNG_URL
-
-  [searxng-ubu2110]      SEARXNG_URL          : http://n.n.n.170/searxng
-  [searxng-ubu2004]      SEARXNG_URL          : http://n.n.n.160/searxng
-  [searxnggfedora35]     SEARXNG_URL          : http://n.n.n.150/searxng
-  [searxng-archlinux]    SEARXNG_URL          : http://n.n.n.140/searxng
-
-Clean up
---------
-
-If there comes the time you want to **get rid off all** the containers and
-**clean up local images** just type::
-
-  $ sudo -H ./utils/lxc.sh remove
-  $ sudo -H ./utils/lxc.sh remove images
-
-
-.. _Setup SearXNG buildhost:
-
-Setup SearXNG buildhost
-=======================
-
-You can **install the SearXNG buildhost environment** into one or all containers.
-The installation procedure to set up a :ref:`build host<buildhosts>` takes its
-time.  Installation in all containers will take more time (time for another cup
-of coffee). ::
-
-  sudo -H ./utils/lxc.sh cmd -- ./utils/searxng.sh install buildhost
-
-To build (live) documentation inside a archlinux_ container::
-
-  sudo -H ./utils/lxc.sh cmd searxng-archlinux make docs.clean docs.live
-  ...
-  [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080
-
-To get IP of the container and the port number *live docs* is listening::
-
-  $ sudo ./utils/lxc.sh show suite | grep docs.live
-  ...
-  [searxng-archlinux]  INFO:  (eth0) docs.live:  http://n.n.n.140:8080/
-
-
-.. _lxc.sh help:
-
-Command Help
-============
-
-The ``--help`` output of the script is largely self-explanatory:
-
-.. program-output:: ../utils/lxc.sh --help
-
-
-.. _lxc-searxng.env:
-
-SearXNG suite config
-====================
-
-The SearXNG suite is defined in the file :origin:`utils/lxc-searxng.env`:
-
-.. literalinclude:: ../../utils/lxc-searxng.env
-   :language: bash

go.mod

@@ -0,0 +1,14 @@
+module searxng.org/devtools
+
+go 1.24.5
+
+tool mvdan.cc/sh/v3/cmd/shfmt
+
+require (
+	github.com/google/renameio/v2 v2.0.0 // indirect
+	github.com/rogpeppe/go-internal v1.14.1 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	mvdan.cc/editorconfig v0.3.0 // indirect
+	mvdan.cc/sh/v3 v3.12.0 // indirect
+)

go.sum

@@ -0,0 +1,22 @@
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
+github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
+golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
+golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
+mvdan.cc/editorconfig v0.3.0 h1:D1D2wLYEYGpawWT5SpM5pRivgEgXjtEXwC9MWhEY0gQ=
+mvdan.cc/editorconfig v0.3.0/go.mod h1:NcJHuDtNOTEJ6251indKiWuzK6+VcrMuLzGMLKBFupQ=
+mvdan.cc/sh/v3 v3.12.0 h1:ejKUR7ONP5bb+UGHGEG/k9V5+pRVIyD+LsZz7o8KHrI=
+mvdan.cc/sh/v3 v3.12.0/go.mod h1:Se6Cj17eYSn+sNooLZiEUnNNmNxg0imoYlTu4CyaGyg=

manage

@@ -32,8 +32,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"
 # shellcheck source=utils/lib_sxng_test.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"
 
-# shellcheck source=utils/lib_go.sh
+# shellcheck source=utils/lib_govm.sh
-source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_govm.sh"
 
 # shellcheck source=utils/lib_valkey.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"
@@ -41,13 +41,14 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"
 # shellcheck source=utils/lib_sxng_vite.sh
 source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh"
 
-PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
+# add ./local dev tools from python (virtualenv), golang and nodejs
+PATH="${PY_ENV}/bin:${REPO_ROOT}/node_modules/.bin:${GOROOT}/bin:${GOPATH}/bin:${PATH}"
 
 # config
 
 PYOBJECTS="searx"
 PY_SETUP_EXTRAS='[test]'
-GECKODRIVER_VERSION="v0.35.0"
+GECKODRIVER_VERSION="v0.36.0"
 # SPHINXOPTS=
 BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
 BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
@@ -63,6 +64,12 @@ RST_FILES=(
     'README.rst'
 )
 
+SHFMT_SCRIPTS=(
+    "./manage"
+    "./container"
+    "./utils"
+)
+
 help() {
     nvm.help
     cat <<EOF
@@ -89,6 +96,7 @@ pyenv.:
   OK        : test if virtualenv is OK
 format.:
   python    : format Python code source using black
+  shell     : format Shell scripts using shfmt
 EOF
     go.help
     node.help
@@ -100,11 +108,26 @@ EOF
     static.help
     vite.help
     cat <<EOF
+dev.:
+  env: enter developer environment (or exec a command in)
 environment ...
   SEARXNG_VALKEY_URL : ${SEARXNG_VALKEY_URL}
 EOF
 }
 
+dev.env() {
+    go.env.dev
+    node.env.dev
+
+    export GOENV
+
+    if [ -z "$1" ]; then
+        export PS1="(dev.env)$ "
+        bash --norc --noprofile
+    else
+        "$@"
+    fi
+}
 
 if [ "$VERBOSE" = "1" ]; then
     SPHINX_VERBOSE="-v"
@@ -148,7 +171,8 @@ gecko.driver() {
 
     build_msg INSTALL "gecko.driver"
     # run installation in a subprocess and activate pyenv
-    (   set -e
+    (
+        set -e
         pyenv.activate
 
         INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
@@ -165,7 +189,7 @@ gecko.driver() {
             "windows 64 bit") ARCH="win64" ;;
             "mac 64bit") ARCH="macos" ;;
         esac
-        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
+        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"
 
         build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
 
@@ -186,7 +210,8 @@ py.build() {
 
 py.clean() {
     build_msg CLEAN pyenv
-    (   set -e
+    (
+        set -e
         pyenv.drop
         [ "$VERBOSE" = "1" ] && set -x
         rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
@@ -212,7 +237,8 @@ pyenv.install() {
         return 0
     fi
 
-    (   set -e
+    (
+        set -e
         pyenv
         build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'"
         "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}"
@@ -225,17 +251,23 @@ pyenv.install() {
 
 pyenv.uninstall() {
     build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
-    pyenv.cmd python setup.py develop --uninstall 2>&1 \
+    pyenv.cmd python setup.py develop --uninstall 2>&1 |
-        | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+        prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
 
 }
 
 format.python() {
-    build_msg TEST "[format.python] black \$BLACK_TARGETS"
+    build_msg TEST "[format.python] black ${BLACK_TARGETS[*]}"
     pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
     dump_return $?
 }
 
+format.shell() {
+    build_msg TEST "[shfmt] shfmt ${SHFMT_SCRIPTS[*]}"
+    go.tool shfmt --list --write "${SHFMT_SCRIPTS[@]}"
+    dump_return $?
+}
+
 docs.prebuild() {
     build_msg DOCS "build ${DOCS_BUILD}/includes"
     (
@@ -252,7 +284,8 @@ docs.prebuild() {
 main() {
 
     local _type
-    local cmd="$1"; shift
+    local cmd="$1"
+    shift
 
     if [ "$cmd" == "" ]; then
         help
@@ -261,7 +294,10 @@ main() {
     fi
 
     case "$cmd" in
-        --getenv) var="$1"; echo "${!var}";;
+        --getenv)
+            var="$1"
+            echo "${!var}"
+            ;;
         --help) help ;;
         --*)
             help

package.json

@@ -1,6 +1,7 @@
 {
+  "name": "searxng.org/devtools",
   "dependencies": {
-    "pyright": "^1.1.391"
+    "pyright": "^1.1.403"
   },
   "scripts": {
     "clean": "rm -Rf node_modules package-lock.json"

pyrightconfig-ci.json

@@ -1,10 +0,0 @@
-{
-    "venvPath": "local",
-    "venv": "py3",
-    "include": [
-        "searx",
-        "searxng_extra",
-        "tests"
-    ],
-    "typeCheckingMode": "off"
-}

searx/data/tracker_patterns.py

@@ -10,6 +10,8 @@ import re
 from collections.abc import Iterator
 from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
 
+from httpx import HTTPError
+
 from searx.data.core import get_cache, log
 from searx.network import get as http_get
 
@@ -70,10 +72,19 @@ class TrackerPatternsDB:
     def iter_clear_list(self) -> Iterator[RuleType]:
         resp = None
         for url in self.CLEAR_LIST_URL:
+            log.debug("TRACKER_PATTERNS: Trying to fetch %s...", url)
+            try:
                 resp = http_get(url, timeout=3)
-            if resp.status_code == 200:
+
-                break
+            except HTTPError as exc:
+                log.warning("TRACKER_PATTERNS: HTTPError (%s) occured while fetching %s", url, exc)
+                continue
+
+            if resp.status_code != 200:
                 log.warning(f"TRACKER_PATTERNS: ClearURL ignore HTTP {resp.status_code} {url}")
+                continue
+
+            break
 
         if resp is None:
             log.error("TRACKER_PATTERNS: failed fetching ClearURL rule lists")

utils/lib.sh

@@ -4,11 +4,20 @@
 # shellcheck disable=SC2059,SC1117
 
 # ubuntu, debian, arch, fedora, centos ...
-DIST_ID=$(source /etc/os-release; echo "$ID");
+DIST_ID=$(
+    source /etc/os-release
+    echo "$ID"
+)
 # shellcheck disable=SC2034
-DIST_VERS=$(source /etc/os-release; echo "$VERSION_ID");
+DIST_VERS=$(
+    source /etc/os-release
+    echo "$VERSION_ID"
+)
 # shellcheck disable=SC2034
-DIST_VERSION_CODENAME=$(source /etc/os-release; echo "$VERSION_CODENAME");
+DIST_VERSION_CODENAME=$(
+    source /etc/os-release
+    echo "$VERSION_CODENAME"
+)
 
 ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
 ADMIN_NAME="${ADMIN_NAME:-$USER}"
@@ -150,12 +159,12 @@ rst_para() {
 }
 
 die() {
-    echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${2-died ${1-1}}" >&2;
+    echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${2-died ${1-1}}" >&2
     exit "${1-1}"
 }
 
 die_caller() {
-    echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[2]}: line ${BASH_LINENO[1]}: ${FUNCNAME[1]}(): ${2-died ${1-1}}" >&2;
+    echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[2]}: line ${BASH_LINENO[1]}: ${FUNCNAME[1]}(): ${2-died ${1-1}}" >&2
     exit "${1-1}"
 }
 
@@ -233,7 +242,8 @@ ask_yn() {
         # shellcheck disable=SC2086,SC2229
         read -r -n1 $_t
         if [[ -z $REPLY ]]; then
-            printf "$default\n"; break
+            printf "$default\n"
+            break
         elif [[ $REPLY =~ ^[Yy]$ ]]; then
             exit_val=${EXIT_YES}
             printf "\n"
@@ -260,7 +270,7 @@ tee_stderr () {
     #   >>> print("hello")
     #    hello
 
-    local _t="0";
+    local _t="0"
     if [[ -n $1 ]]; then _t="$1"; fi
 
     (while read -r line; do
@@ -320,9 +330,11 @@ cache_download() {
         info_msg "caching: $1"
         info_msg "  --> ${CACHE}/$2"
         if [[ -n ${SUDO_USER} ]]; then
-            sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
+            sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1"
+            exit_value=$?
         else
-            wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
+            wget --progress=bar -O "${CACHE}/$2" "$1"
+            exit_value=$?
         fi
         if [[ ! $exit_value = 0 ]]; then
             err_msg "failed to download: $1"
@@ -350,7 +362,7 @@ choose_one() {
     local default=${DEFAULT_SELECT-1}
     local REPLY
     local env_name=$1 && shift
-    local choice=$1;
+    local choice=$1
     local max="${#@}"
     local _t
     [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
@@ -414,8 +426,14 @@ install_template() {
 
     for i in "$@"; do
         case $i in
-            --no-eval) do_eval=0; shift ;;
+            --no-eval)
-            --variant=*) variant=":${i#*=}"; shift ;;
+                do_eval=0
+                shift
+                ;;
+            --variant=*)
+                variant=":${i#*=}"
+                shift
+                ;;
             *) pos_args+=("$i") ;;
         esac
     done
@@ -503,6 +521,7 @@ install_template() {
                 ;;
             "diff files")
                 $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout
+                ;;
         esac
     done
 }
@@ -563,8 +582,8 @@ pyenv() {
     # files.
 
     required_commands \
-        sha256sum "${PYTHON}" \
+        sha256sum "${PYTHON}" ||
-        || exit
+        exit
 
     local pip_req=()
 
@@ -583,8 +602,8 @@ pyenv() {
         (
             [ "$VERBOSE" = "1" ] && set -x
             # shellcheck disable=SC2086
-            "${PY_ENV_BIN}/python" -m pip install "${pip_req[@]}" \
+            "${PY_ENV_BIN}/python" -m pip install "${pip_req[@]}" &&
-                && sha256sum ${PY_ENV_REQ} > "${PY_ENV}/requirements.sha256"
+                sha256sum ${PY_ENV_REQ} >"${PY_ENV}/requirements.sha256"
         )
     fi
     pyenv.OK
@@ -602,17 +621,17 @@ pyenv.OK() {
         return 1
     fi
 
-    if [ ! -f "${PY_ENV}/requirements.sha256" ] \
+    if [ ! -f "${PY_ENV}/requirements.sha256" ] ||
-        || ! sha256sum -c "${PY_ENV}/requirements.sha256" > /dev/null 2>&1; then
+        ! sha256sum -c "${PY_ENV}/requirements.sha256" >/dev/null 2>&1; then
         build_msg PYENV "[virtualenv] requirements.sha256 failed"
         sed 's/^/          [virtualenv] - /' <"${PY_ENV}/requirements.sha256"
         return 1
     fi
 
     if [ "$VERBOSE" = "1" ]; then
-        pyenv.check \
+        pyenv.check |
-            | "${PY_ENV_BIN}/python" 2>&1 \
+            "${PY_ENV_BIN}/python" 2>&1 |
-            | prefix_stdout "${_Blue}PYENV     ${_creset}[check] "
+            prefix_stdout "${_Blue}PYENV     ${_creset}[check] "
     else
         pyenv.check | "${PY_ENV_BIN}/python" 1>/dev/null
     fi
@@ -708,19 +727,19 @@ pyenv.uninstall() {
     build_msg PYENV "[uninstall] ${PYOBJECTS}"
 
     if [ "." = "${PYOBJECTS}" ]; then
-	pyenv.cmd python setup.py develop --uninstall 2>&1 \
+        pyenv.cmd python setup.py develop --uninstall 2>&1 |
-            | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+            prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
     else
         # shellcheck disable=SC2086
-	pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 \
+        pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 |
-            | prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
+            prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "
     fi
 }
 
-
 pyenv.cmd() {
     pyenv.install
-    (   set -e
+    (
+        set -e
         # shellcheck source=/dev/null
         source "${PY_ENV_BIN}/activate"
         [ "$VERBOSE" = "1" ] && set -x
@@ -728,14 +747,12 @@ pyenv.cmd() {
     )
 }
 
-
 pyenv.activate() {
     pyenv.install
     # shellcheck source=/dev/null
     source "${PY_ENV_BIN}/activate"
 }
 
-
 # Sphinx doc
 # ----------
 
@@ -858,7 +875,6 @@ interactive_shell(){
     sudo -H -u "${1}" -i
 }
 
-
 # systemd
 # -------
 
@@ -927,7 +943,6 @@ systemctl status --no-pager ${1}.service
 EOF
 }
 
-
 # nginx
 # -----
 
@@ -1042,8 +1057,7 @@ nginx_include_apps_enabled() {
     (
         local line
         local stage=0
-        while IFS=  read -r line
+        while IFS= read -r line; do
-        do
             echo "$line"
             if [[ $stage = 0 ]]; then
                 if [[ $line =~ ^[[:space:]]*server*[[:space:]]*\{ ]]; then
@@ -1095,7 +1109,6 @@ nginx_disable_app() {
     nginx_reload
 }
 
-
 # Apache
 # ------
 
@@ -1626,147 +1639,6 @@ EOF
     fi
 }
 
-# containers
-# ----------
-
-in_container() {
-    # Test if shell runs in a container.
-    #
-    # usage:  in_container && echo "process running inside a LXC container"
-    #         in_container || echo "process is not running inside a LXC container"
-    #
-    # sudo_or_exit
-    # hint:   Reads init process environment, therefore root access is required!
-    # to be safe, take a look at the environment of process 1 (/sbin/init)
-    # grep -qa 'container=lxc' /proc/1/environ
-
-    # see lxc_init_container_env
-    [[ -f /.lxcenv ]]
-}
-
-LXC_ENV_FOLDER=
-if in_container; then
-    # shellcheck disable=SC2034
-    LXC_ENV_FOLDER="lxc-env/$(hostname)/"
-    PY_ENV="${LXC_ENV_FOLDER}${PY_ENV}"
-    PY_ENV_BIN="${LXC_ENV_FOLDER}${PY_ENV_BIN}"
-    PYDIST="${LXC_ENV_FOLDER}${PYDIST}"
-    PYBUILD="${LXC_ENV_FOLDER}${PYBUILD}"
-    DOCS_DIST="${LXC_ENV_FOLDER}${DOCS_DIST}"
-    DOCS_BUILD="${LXC_ENV_FOLDER}${DOCS_BUILD}"
-fi
-
-lxc_init_container_env() {
-
-    # usage: lxc_init_container_env <name>
-
-    # Create a /.lxcenv file in the root folder.  Call this once after the
-    # container is initial started and before installing any boilerplate stuff.
-
-    info_msg "create /.lxcenv in container $1"
-    cat <<EOF | lxc exec "${1}" -- bash | prefix_stdout "[${_BBlue}${1}${_creset}] "
-touch "/.lxcenv"
-ls -l "/.lxcenv"
-EOF
-}
-
-# apt packages
-LXC_BASE_PACKAGES_debian="bash git build-essential python3 python3-venv python-is-python3"
-
-# pacman packages
-LXC_BASE_PACKAGES_arch="bash git base-devel python"
-
-# dnf packages
-LXC_BASE_PACKAGES_fedora="bash git @development-tools python"
-
-# yum packages
-LXC_BASE_PACKAGES_centos="bash git python3"
-
-lxc_distro_setup() {
-    case $DIST_ID in
-        ubuntu|debian) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_debian}" ;;
-        arch)          LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_arch}" ;;
-        fedora)        LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_fedora}" ;;
-        centos)        LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_centos}" ;;
-        *) err_msg "$DIST_ID-$DIST_VERS: pkg_install LXC_BASE_PACKAGES not yet implemented" ;;
-    esac
-}
-
-lxc_install_base_packages() {
-    info_msg "install LXC_BASE_PACKAGES in container $1"
-    case $DIST_ID in
-        centos) yum groupinstall "Development Tools" -y  ;;
-    esac
-    pkg_install "${LXC_BASE_PACKAGES}"
-}
-
-
-lxc_image_copy() {
-
-    # usage: lxc_image_copy <remote image> <local image>
-    #
-    #        lxc_image_copy "images:ubuntu/20.04"  "ubu2004"
-
-    if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then
-        info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}"
-    else
-        info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}"
-        lxc image copy "${LXC_SUITE[i]}" local: \
-            --alias  "${LXC_SUITE[i+1]}" | prefix_stdout
-    fi
-}
-
-lxc_init_container() {
-
-    # usage: lxc_init_container <image name> <container name>
-
-    local image_name="$1"
-    local container_name="$2"
-
-    if lxc info "${container_name}" &>/dev/null; then
-        info_msg "container '${container_name}' already exists"
-    else
-        info_msg "create container instance: ${container_name}"
-        lxc init "local:${image_name}" "${container_name}"
-    fi
-}
-
-lxc_exists(){
-
-    # usage: lxc_exists <name> || echo "container <name> does not exists"
-
-    lxc info "$1" &>/dev/null
-}
-
-lxc_image_exists(){
-    # usage: lxc_image_exists <alias> || echo "image <alias> does locally not exists"
-
-    lxc image info "local:$1" &>/dev/null
-
-}
-
-lxc_delete_container() {
-
-    #  usage: lxc_delete_container <container-name>
-
-    if lxc info "$1" &>/dev/null; then
-        info_msg "stop & delete instance ${_BBlue}${1}${_creset}"
-        lxc stop "$1" &>/dev/null
-        lxc delete "$1" | prefix_stdout
-    else
-        warn_msg "instance '$1' does not exist / can't delete :o"
-    fi
-}
-
-lxc_delete_local_image() {
-
-    #  usage: lxc_delete_local_image <container-name>
-
-    info_msg "delete image 'local:$i'"
-    lxc image delete "local:$i"
-}
-
-
 # IP
 # --
 
@@ -1787,9 +1659,9 @@ primary_ip() {
 
     case $DIST_ID in
         arch)
-            ip -o addr show \
+            ip -o addr show |
-                | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \
+                sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' |
-                | head -n 1
+                head -n 1
             ;;
         *) hostname -I | cut -d' ' -f1 ;;
     esac

utils/lib_go.sh

@@ -1,211 +0,0 @@
-#!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
-# SPDX-License-Identifier: AGPL-3.0-or-later
-#
-# Tools to install and maintain golang [1] binaries & packages.
-#
-# [1] https://golang.org/doc/devel/release#policy
-#
-# A simple *helloworld* test with user 'my_user' :
-#
-#   sudo -H adduser my_user
-#   ./manage go.golang go1.17.3 my_user
-#   ./manage go.install github.com/go-training/helloworld@latest my_user
-#   ./manage go.bash my_user
-#   $ helloword
-#   Hello World!!
-#
-# Don't forget to remove 'my_user':  sudo -H deluser --remove-home my_user
-
-# shellcheck source=utils/lib.sh
-. /dev/null
-
-# configure golang environment
-# ----------------------------
-
-[[ -z "${GO_VERSION}" ]] && GO_VERSION="go1.17.3"
-
-GO_DL_URL="https://golang.org/dl"
-
-# implement go functions
-# -----------------------
-
-go.help(){
-    cat <<EOF
-go.:
-  ls        : list golang binary archives (stable)
-  golang    : (re-) install golang binary in user's \$HOME/local folder
-  install   : install go package in user's \$HOME/go-apps folder
-  bash      : start bash interpreter with golang environment sourced
-EOF
-}
-
-go.ls(){
-    python <<EOF
-import sys, json, requests
-resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
-for ver in json.loads(resp.text):
-    if not ver['stable']:
-        continue
-    for f in ver['files']:
-        if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:
-            continue
-        print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)
-EOF
-}
-
-go.ver_info(){
-
-    # print information about a golang distribution. To print filename
-    # sha256 and size of the archive that fits to your OS and host:
-    #
-    #   go.ver_info "${GO_VERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
-    #
-    # usage: go.ver_info <go-vers> <kind> <os> <arch> [filename|sha256|size]
-    #
-    # kind:  [archive|source|installer]
-    # os:    [darwin|freebsd|linux|windows]
-    # arch:  [amd64|arm64|386|armv6l|ppc64le|s390x]
-
-    python - "$@" <<EOF
-import sys, json, requests
-resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
-for ver in json.loads(resp.text):
-    if ver['version'] != sys.argv[1]:
-        continue
-    for f in ver['files']:
-        if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):
-            continue
-        for x in sys.argv[5:]:
-           print(f[x])
-        sys.exit(0)
-sys.exit(42)
-EOF
-}
-
-go.os() {
-  local OS
-  case "$(command uname -a)xx" in
-    Linux\ *) OS=linux ;;
-    Darwin\ *) OS=darwin ;;
-    FreeBSD\ *) OS=freebsd ;;
-    CYGWIN* | MSYS* | MINGW*) OS=windows ;;
-    *)  die 42 "OS is unknown: $(command uname -a)" ;;
-  esac
-  echo "${OS}"
-}
-
-go.arch() {
-    local ARCH
-    case "$(command uname -m)" in
-        "x86_64") ARCH=amd64 ;;
-        "aarch64") ARCH=arm64 ;;
-        "armv6" | "armv7l") ARCH=armv6l ;;
-        "armv8") ARCH=arm64 ;;
-        .*386.*) ARCH=386 ;;
-        ppc64*) ARCH=ppc64le ;;
-    *)  die 42 "ARCH is unknown: $(command uname -m)" ;;
-    esac
-    echo "${ARCH}"
-}
-
-go.golang() {
-
-    # install golang binary in user's $HOME/local folder:
-    #
-    #   go.golang ${GO_VERSION} ${SERVICE_USER}
-    #
-    # usage:  go.golang <go-vers> [<username>]
-
-    local version fname sha size user userpr
-    local buf=()
-
-    version="${1:-${GO_VERSION}}"
-    user="${2:-${USERNAME}}"
-    userpr="  ${_Yellow}|${user}|${_creset} "
-
-    rst_title "Install Go in ${user}'s HOME" section
-
-    mapfile -t buf < <(
-        go.ver_info "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size
-    )
-
-    if [ ${#buf[@]} -eq 0 ]; then
-        die 42 "can't find info of golang version: ${version}"
-    fi
-    fname="${buf[0]}"
-    sha="${buf[1]}"
-    size="$(numfmt --to=iec "${buf[2]}")"
-
-    info_msg "Download go binary ${fname} (${size}B)"
-    cache_download "${GO_DL_URL}/${fname}" "${fname}"
-
-    pushd "${CACHE}" &> /dev/null
-    echo "${sha}  ${fname}" > "${fname}.sha256"
-    if ! sha256sum -c "${fname}.sha256" >/dev/null; then
-        die 42 "downloaded file ${fname} checksum does not match"
-    else
-        info_msg "${fname} checksum OK"
-    fi
-    popd &> /dev/null
-
-    info_msg "install golang"
-    tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
-mkdir -p \$HOME/local
-rm -rf \$HOME/local/go
-tar -C \$HOME/local -xzf ${CACHE}/${fname}
-echo "export GOPATH=\$HOME/go-apps" > \$HOME/.go_env
-echo "export PATH=\$HOME/local/go/bin:\\\$GOPATH/bin:\\\$PATH" >> \$HOME/.go_env
-EOF
-    info_msg "test golang installation"
-    sudo -i -u "${user}" <<EOF
-source \$HOME/.go_env
-command -v go
-go version
-EOF
-}
-
-go.install() {
-
-    # install go package in user's $HOME/go-apps folder:
-    #
-    #   go.install github.com/go-training/helloworld@lates ${SERVICE_USER}
-    #
-    # usage:  go.install <package> [<username>]
-
-    local package user userpr
-
-    package="${1}"
-    user="${2:-${USERNAME}}"
-    userpr="  ${_Yellow}|${user}|${_creset} "
-
-    if [ -z "${package}" ]; then
-        die 42 "${FUNCNAME[0]}() - missing argument: <package>"
-    fi
-    tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
-source \$HOME/.go_env
-go install -v ${package}
-EOF
-}
-
-go.bash() {
-
-    # start bash interpreter with golang environment sourced
-    #
-    #   go.bash ${SERVICE_USER}
-    #
-    # usage:  go.bash [<username>]
-
-    local user
-    user="${1:-${USERNAME}}"
-    sudo -i -u "${user}" bash --init-file "~${user}/.go_env"
-}
-
-go.version(){
-    local user
-    user="${1:-${USERNAME}}"
-    sudo -i -u "${user}" <<EOF
-source \$HOME/.go_env
-go version | cut -d' ' -f 3
-EOF
-}

utils/lib_govm.sh

@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-or-later
+#
+# Go versions manager to install and maintain golang [1] binaries & packages.
+#
+# [1] https://golang.org/doc/devel/release#policy
+
+# shellcheck source=utils/lib.sh
+. /dev/null
+
+# configure golang environment for go.vm
+# --------------------------------------
+
+_GO_DL_URL="https://go.dev/dl"
+
+GOVERSION="${GOVERSION:-go$(awk '/^go /{print $2}' "${REPO_ROOT}/go.mod")}"
+GOTOOLCHAIN=local
+
+GOROOT="${REPO_ROOT}/.govm/${GOVERSION}"
+GOENV="${GOROOT}/.config/go.env"
+GOVM_EXE="${GOROOT}/bin/go"
+
+GOPATH="${REPO_ROOT}/local/${GOVERSION}" # no support for multiple path names!!
+GOCACHE="${GOPATH}/.cache/go-build"
+GOMODCACHE="${GOPATH}/pkg/mod"
+
+# implement go functions
+# -----------------------
+
+go.help() {
+    cat <<EOF
+go:           GOROOT=${GOROOT}
+  install   : compiles and installs packages
+EOF
+}
+
+go.tool() {
+    # shortcut for "go tool .." in the Go environment
+    go.env.dev
+    "${GOVM_EXE}" tool "$@"
+}
+
+go.env.dev() {
+    if [ -z "$_GO_DEVTOOLS_INSTALLED" ]; then
+        build_msg INSTALL "[pkg.go.dev] ./go.mod: developer and CI tools"
+        go.tidy
+    else
+        go.vm.ensure
+        _GO_DEVTOOLS_INSTALLED=1
+    fi
+}
+
+go.tidy() {
+    go.vm.ensure
+    "${GOVM_EXE}" mod tidy
+    chmod -R u+w "${GOMODCACHE}"
+}
+
+go.clean() {
+    if ! go.vm.is_installed; then
+        build_msg CLEAN "[Go] not installed"
+        return 0
+    fi
+    build_msg CLEAN "[Go] drop folders ${GOROOT} and ${GOPATH}"
+    rm -rf "${GOROOT}" "${GOPATH}"
+}
+
+go.install() {
+    go.vm.ensure
+    GOENV="${GOENV}" "${GOVM_EXE}" install "$@"
+    # not sure why, but go installs some files without setting the write access
+    # for the file owner
+    chmod -R u+w "${GOMODCACHE}"
+}
+
+go.os() {
+    local OS
+    case "$(command uname -a)xx" in
+        Linux\ *) OS=linux ;;
+        Darwin\ *) OS=darwin ;;
+        FreeBSD\ *) OS=freebsd ;;
+        CYGWIN* | MSYS* | MINGW*) OS=windows ;;
+        *) die 42 "OS is unknown: $(command uname -a)" ;;
+    esac
+    echo "${OS}"
+}
+
+go.arch() {
+    local ARCH
+    case "$(command uname -m)" in
+        "x86_64") ARCH=amd64 ;;
+        "aarch64") ARCH=arm64 ;;
+        "armv6" | "armv7l") ARCH=armv6l ;;
+        "armv8") ARCH=arm64 ;;
+        .*386.*) ARCH=386 ;;
+        ppc64*) ARCH=ppc64le ;;
+        *) die 42 "ARCH is unknown: $(command uname -m)" ;;
+    esac
+    echo "${ARCH}"
+}
+
+# Go version management (go.vm)
+# -----------------------------
+
+go.vm.ensure() {
+    if ! go.vm.is_installed; then
+        # shellcheck disable=SC2119
+        go.vm.install
+    fi
+}
+
+go.vm.is_installed() {
+    # is true if "go" command is installed
+    [[ -f "${GOROOT}/bin/go" ]]
+}
+
+# shellcheck disable=SC2120
+go.vm.install() {
+
+    # Go versions manager; to install Go at arbitrary place:
+    #
+    # usage:  go.vm.install <version> <dest>
+
+    local version dest fname sha size tmp
+    version="${1:-$GOVERSION}"
+    dest="${2:-$GOROOT}"
+
+    info_msg "Install Go in ${dest}"
+
+    # HINT: the python requirements needed by go.vm.version are taken from the
+    # developer environment. If it is not yet installed, install it now ..
+    pyenv.install
+
+    # fetch go version ..
+    local buf=()
+    mapfile -t buf < <(
+        go.vm.version "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size
+    )
+    if [ ${#buf[@]} -eq 0 ]; then
+        die 42 "can't find info of golang version: ${version}"
+    fi
+    fname="${buf[0]}"
+    sha="${buf[1]}"
+    size="$(numfmt --to=iec "${buf[2]}")"
+
+    info_msg "Download go binary ${fname} (${size}B)"
+    cache_download "${_GO_DL_URL}/${fname}" "${fname}"
+
+    pushd "${CACHE}" &>/dev/null
+    echo "${sha}  ${fname}" >"${fname}.sha256"
+    if ! sha256sum -c "${fname}.sha256" >/dev/null; then
+        die 42 "downloaded file ${fname} checksum does not match"
+    else
+        info_msg "${fname} checksum OK"
+    fi
+    popd &>/dev/null
+
+    info_msg "install golang"
+
+    tmp="$(mktemp -d)"
+    tar -C "${tmp}" -xzf "${CACHE}/${fname}"
+    rm -rf "${dest}"
+    mkdir -p "$(dirname "${dest}")"
+    mv "${tmp}/go" "${dest}"
+
+    mkdir -p "$(dirname "$GOENV")"
+    export GOENV
+
+    "${GOVM_EXE}" telemetry off
+    "${GOVM_EXE}" env -w \
+        GOBIN="$GOBIN" \
+        GOTOOLCHAIN="$GOTOOLCHAIN" \
+        GOCACHE="$GOCACHE" \
+        GOPATH="$GOPATH" \
+        GOMODCACHE="$GOMODCACHE"
+
+    mkdir -p "${GOMODCACHE}"
+}
+
+go.vm.list() {
+
+    # Go versions manager; list Go versions (stable)
+
+    "${PY_ENV_BIN}/python" <<EOF
+import sys, json, requests
+resp = requests.get("${_GO_DL_URL}/?mode=json&include=all")
+for ver in json.loads(resp.text):
+    if not ver['stable']:
+        continue
+    for f in ver['files']:
+        if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:
+            continue
+        print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)
+EOF
+}
+
+go.vm.version() {
+
+    # Print information about a Go distribution. To print filename sha256 and
+    # size of the archive that fits to your OS and host:
+    #
+    #   go.ver_info "${GOVERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
+    #
+    # usage: go.vm.version <go-vers> <kind> <os> <arch> [filename|sha256|size]
+    #
+    # kind:  [archive|source|installer]
+    # os:    [darwin|freebsd|linux|windows]
+    # arch:  [amd64|arm64|386|armv6l|ppc64le|s390x]
+
+    "${PY_ENV_BIN}/python" - "$@" <<EOF
+import sys, json, requests
+resp = requests.get("${_GO_DL_URL}/?mode=json&include=all")
+for ver in json.loads(resp.text):
+    if ver['version'] != sys.argv[1]:
+        continue
+    for f in ver['files']:
+        if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):
+            continue
+        for x in sys.argv[5:]:
+           print(f[x])
+        sys.exit(0)
+sys.exit(42)
+EOF
+}

utils/lib_redis.sh

@@ -21,7 +21,6 @@ redis.:
 EOF
 }
 
-
 redis.remove() {
     sudo_or_exit
     (
@@ -36,7 +35,6 @@ redis.shell() {
     interactive_shell "${REDIS_USER}"
 }
 
-
 redis.userdel() {
     sudo_or_exit
     drop_service_account "${REDIS_USER}"

utils/lib_sxng_data.sh

@@ -13,7 +13,8 @@ EOF
 }
 
 data.all() {
-    (   set -e
+    (
+        set -e
 
         pyenv.activate
         data.traits
@@ -35,9 +36,9 @@ data.all() {
     )
 }
 
-
 data.traits() {
-    (   set -e
+    (
+        set -e
         pyenv.activate
         build_msg DATA "update searx/data/engine_traits.json"
         python searxng_extra/update/update_engine_traits.py
@@ -53,7 +54,8 @@ data.useragents() {
 }
 
 data.locales() {
-    (   set -e
+    (
+        set -e
         pyenv.activate
         build_msg DATA "update searx/data/locales.json"
         python searxng_extra/update/update_locales.py
@@ -62,7 +64,8 @@ data.locales() {
 }
 
 data.currencies() {
-    (   set -e
+    (
+        set -e
         pyenv.activate
         build_msg DATA "update searx/data/currencies.json"
         python searxng_extra/update/update_currencies.py

utils/lib_sxng_node.sh

@@ -24,7 +24,8 @@ nodejs.ensure() {
 
 node.env() {
     nodejs.ensure
-    (   set -e
+    (
+        set -e
         build_msg INSTALL "[npm] ./client/simple/package.json"
         npm --prefix client/simple install
     )
@@ -43,17 +44,19 @@ node.clean() {
         return 0
     fi
     build_msg CLEAN "themes -- locally installed npm dependencies"
-    (   set -e
+    (
-        npm --prefix client/simple run clean \
+        set -e
-	    | prefix_stdout "${_Blue}CLEAN    ${_creset} "
+        npm --prefix client/simple run clean |
+            prefix_stdout "${_Blue}CLEAN    ${_creset} "
         if [ "${PIPESTATUS[0]}" -ne "0" ]; then
             return 1
         fi
     )
     build_msg CLEAN "locally installed developer and CI tools"
-    (   set -e
+    (
-        npm --prefix . run clean \
+        set -e
-	    | prefix_stdout "${_Blue}CLEAN    ${_creset} "
+        npm --prefix . run clean |
+            prefix_stdout "${_Blue}CLEAN    ${_creset} "
         if [ "${PIPESTATUS[0]}" -ne "0" ]; then
             return 1
         fi

utils/lib_sxng_static.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-
 STATIC_BUILD_COMMIT="[build] /static"
 STATIC_BUILT_PATHS=(
     'searx/templates/simple/icons.html'
@@ -96,7 +95,8 @@ static.build.commit() {
     # drop existing commit from previous build
     static.build.drop &>/dev/null
 
-    (   set -e
+    (
+        set -e
         # fix & build the themes
         themes.fix
         themes.lint

utils/lib_sxng_test.sh

@@ -6,8 +6,9 @@ test.help(){
 test.:
   yamllint  : lint YAML files (YAMLLINT_FILES)
   pylint    : lint ./searx, ./searxng_extra and ./tests
-  pyright   : static type check of python sources (.dev or .ci)
+  pyright   : check Python types
-  black     : check black code format
+  black     : check Python code format
+  shfmt     : check Shell script code format
   unit      : run unit tests
   coverage  : run unit tests with coverage
   robot     : run robot test
@@ -21,13 +22,14 @@ if [ "$VERBOSE" = "1" ]; then
 fi
 
 test.yamllint() {
-    build_msg TEST "[yamllint] \$YAMLLINT_FILES"
+    build_msg TEST "[yamllint] $YAMLLINT_FILES"
     pyenv.cmd yamllint --strict --format parsable "${YAMLLINT_FILES[@]}"
     dump_return $?
 }
 
 test.pylint() {
-    (   set -e
+    (
+        set -e
         pyenv.activate
         PYLINT_OPTIONS="--rcfile .pylintrc"
 
@@ -48,63 +50,41 @@ test.pylint() {
     dump_return $?
 }
 
-test.types.dev() {
+test.pyright() {
     # use this pyright test for local tests in development / it suppress
     # warnings related to intentional monkey patching but gives good hints where
     # we need to work on SearXNG's typification.
-    #
-    # --> pyrightconfig.json
 
     build_msg TEST "[pyright/types] static type check of python sources"
-    build_msg TEST "    --> typeCheckingMode: on"
     node.env.dev
 
     build_msg TEST "[pyright/types] suppress warnings related to intentional monkey patching"
     # We run Pyright in the virtual environment because pyright executes
     # "python" to determine the Python version.
-    pyenv.cmd npx --no-install pyright -p pyrightconfig.json \
+    pyenv.cmd npx --no-install pyright -p pyrightconfig.json |
-        | grep -E '\.py:[0-9]+:[0-9]+'\
+        grep -E '\.py:[0-9]+:[0-9]+' |
-        | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
+        grep -v '/engines/.*.py.* - warning: "logger" is not defined' |
-        | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
+        grep -v '/plugins/.*.py.* - error: "logger" is not defined' |
-        | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
+        grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' |
-        | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
+        grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' |
-        | grep -v '/engines/.*.py.* - warning: "categories" is not defined'
+        grep -v '/engines/.*.py.* - warning: "categories" is not defined'
-    # ignore exit value from pyright
-    # dump_return ${PIPESTATUS[0]}
-    return 0
-}
-
-test.types.ci() {
-    # use this pyright test for CI / disables typeCheckingMode, needed as long
-    # we do not have fixed all typification issues.
-    #
-    # --> pyrightconfig-ci.json
-
-    build_msg TEST "[pyright] static type check of python sources"
-    build_msg TEST "    --> typeCheckingMode: off !!!"
-    node.env.dev
-
-    build_msg TEST "[pyright] suppress warnings related to intentional monkey patching"
-    # We run Pyright in the virtual environment because pyright executes
-    # "python" to determine the Python version.
-    pyenv.cmd npx --no-install pyright -p pyrightconfig-ci.json \
-        | grep -E '\.py:[0-9]+:[0-9]+'\
-        | grep -v '/engines/.*.py.* - warning: "logger" is not defined'\
-        | grep -v '/plugins/.*.py.* - error: "logger" is not defined'\
-        | grep -v '/engines/.*.py.* - warning: "supported_languages" is not defined' \
-        | grep -v '/engines/.*.py.* - warning: "language_aliases" is not defined' \
-        | grep -v '/engines/.*.py.* - warning: "categories" is not defined'
     # ignore exit value from pyright
     # dump_return ${PIPESTATUS[0]}
     return 0
 }
 
 test.black() {
-    build_msg TEST "[black] \$BLACK_TARGETS"
+    build_msg TEST "[black] $BLACK_TARGETS"
     pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
     dump_return $?
 }
 
+test.shfmt() {
+    build_msg TEST "[shfmt] ${SHFMT_SCRIPTS[*]}"
+    go.tool shfmt --list --diff "${SHFMT_SCRIPTS[@]}"
+    dump_return $?
+}
+
 test.unit() {
     build_msg TEST 'tests/unit'
     # shellcheck disable=SC2086
@@ -114,7 +94,8 @@ test.unit() {
 
 test.coverage() {
     build_msg TEST 'unit test coverage'
-    (   set -e
+    (
+        set -e
         pyenv.activate
         # shellcheck disable=SC2086
         python -m nose2 ${TEST_NOSE2_VERBOSE} -C --log-capture --with-coverage --coverage searx -s tests/unit

utils/lib_sxng_themes.sh

@@ -13,14 +13,16 @@ EOF
 }
 
 themes.all() {
-    (   set -e
+    (
+        set -e
         vite.simple.build
     )
     dump_return $?
 }
 
 themes.simple() {
-    (   set -e
+    (
+        set -e
         build_msg SIMPLE "theme: run build (simple)"
         vite.simple.build
     )
@@ -28,7 +30,8 @@ themes.simple() {
 }
 
 themes.fix() {
-    (   set -e
+    (
+        set -e
         build_msg SIMPLE "theme: fix (all themes)"
         vite.simple.fix
     )
@@ -36,7 +39,8 @@ themes.fix() {
 }
 
 themes.lint() {
-    (   set -e
+    (
+        set -e
         build_msg SIMPLE "theme: lint (all themes)"
         vite.simple.lint
     )
@@ -44,7 +48,8 @@ themes.lint() {
 }
 
 themes.test() {
-    (   set -e
+    (
+        set -e
         # we run a build to test (in CI)
         build_msg SIMPLE "theme: run build (to test)"
         vite.simple.build

utils/lib_sxng_vite.sh

@@ -30,7 +30,8 @@ VITE_SIMPLE_THEME="${REPO_ROOT}/client/simple"
 # }
 
 vite.simple.build() {
-    (   set -e
+    (
+        set -e
         templates.simple.pygments
 
         node.env
@@ -44,14 +45,16 @@ vite.simple.build() {
 }
 
 vite.simple.fix() {
-    (   set -e
+    (
+        set -e
         node.env
         npm --prefix client/simple run fix
     )
 }
 
 vite.simple.lint() {
-    (   set -e
+    (
+        set -e
         node.env
         npm --prefix client/simple run lint
     )
@@ -59,8 +62,8 @@ vite.simple.lint() {
 
 templates.simple.pygments() {
     build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
-    pyenv.cmd python searxng_extra/update/update_pygments.py \
+    pyenv.cmd python searxng_extra/update/update_pygments.py |
-        | prefix_stdout "${_Blue}PYGMENTS ${_creset} "
+        prefix_stdout "${_Blue}PYGMENTS ${_creset} "
     if [ "${PIPESTATUS[0]}" -ne "0" ]; then
         build_msg PYGMENTS "building LESS files for pygments failed"
         return 1

utils/lib_sxng_weblate.sh

@@ -19,7 +19,8 @@ weblate.translations.worktree() {
     #
     #     remote weblate https://translate.codeberg.org/git/searxng/searxng/
 
-    (   set -e
+    (
+        set -e
         if ! git remote get-url weblate 2>/dev/null; then
             git remote add weblate https://translate.codeberg.org/git/searxng/searxng/
         fi
@@ -49,7 +50,8 @@ weblate.to.translations() {
     # 4. In translations worktree, merge changes of branch 'translations' from
     #    remote 'weblate' and push it on branch 'translations' of 'origin'
 
-    (   set -e
+    (
+        set -e
         pyenv.activate
         if [ "$(wlc lock-status)" != "locked: True" ]; then
             die 1 "weblate must be locked, currently: $(wlc lock-status)"
@@ -77,14 +79,18 @@ weblate.translations.commit() {
     # create a commit in the local branch (master)
 
     local existing_commit_hash commit_body commit_message exitcode
-    (   set -e
+    (
+        set -e
         pyenv.activate
         # lock change on weblate
         wlc lock
 
         # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
         weblate.translations.worktree
-        existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1  --pretty=format:'%h')
+        existing_commit_hash=$(
+            cd "${TRANSLATIONS_WORKTREE}"
+            git log -n1 --pretty=format:'%h'
+        )
 
         # pull weblate commits
         weblate.to.translations
@@ -101,7 +107,10 @@ weblate.translations.commit() {
         data.locales
 
         # git add/commit (no push)
-        commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
+        commit_body=$(
+            cd "${TRANSLATIONS_WORKTREE}"
+            git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD"
+        )
         commit_message=$(echo -e "[l10n] update translations from Weblate\n\n${commit_body}")
         git add searx/translations
         git add searx/data/locales.json
@@ -135,7 +144,8 @@ weblate.push.translations() {
     local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
         exitcode
     messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
-    (   set -e
+    (
+        set -e
         pyenv.activate
         # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
         weblate.translations.worktree
@@ -147,8 +157,10 @@ weblate.push.translations() {
             "searx/"
 
         # stop if there is no meaningful change in the master branch
-        diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
+        diff_messages_pot=$(
-                            git diff -- "searx/translations/messages.pot")
+            cd "${TRANSLATIONS_WORKTREE}"
+            git diff -- "searx/translations/messages.pot"
+        )
         if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
             build_msg BABEL 'no changes detected, exiting'
             return 42

utils/lxc-searxng.env

@@ -1,69 +0,0 @@
-# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
-# SPDX-License-Identifier: AGPL-3.0-or-later
-# shellcheck shell=bash
-
-# This file is a setup of a LXC suite.  It is sourced from different context, do
-# not manipulate the environment directly, implement functions and manipulate
-# environment only in subshells.
-
-lxc_set_suite_env() {
-
-    export LXC_SUITE_NAME="searxng"
-
-    # name of https://images.linuxcontainers.org
-    export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}"
-    export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}"
-    export LXC_SUITE=(
-
-        # end of standard support see https://wiki.ubuntu.com/Releases
-        "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04"  "ubu2004" # LTS EOSS April 2025
-        "$LINUXCONTAINERS_ORG_NAME:ubuntu/22.04"  "ubu2204" # LTS EOSS April 2027
-
-        # EOL see https://fedoraproject.org/wiki/Releases
-        "$LINUXCONTAINERS_ORG_NAME:fedora/35"     "fedora35"
-
-        # rolling releases see https://www.archlinux.org/releng/releases/
-        "$LINUXCONTAINERS_ORG_NAME:archlinux"     "archlinux"
-    )
-}
-
-lxc_suite_install_info() {
-    (
-        lxc_set_suite_env
-        cat <<EOF
-LXC suite: ${LXC_SUITE_NAME}
-  Suite includes installation of SearXNG
-  images:     ${LOCAL_IMAGES[*]}
-  containers: ${CONTAINERS[*]}
-EOF
-    )
-}
-
-lxc_suite_install() {
-    (
-        lxc_set_suite_env
-        FORCE_TIMEOUT=0 "${LXC_REPO_ROOT}/utils/searxng.sh" install all
-        rst_title "Suite installation finished ($(hostname))" part
-        if ask_yn "Developer install? (wraps source from HOST into the running instance)" Yn; then
-            "${LXC_REPO_ROOT}/utils/searxng.sh" searxng.install.link_src "$(pwd)"
-        fi
-        lxc_suite_info
-        echo
-    )
-}
-
-lxc_suite_info() {
-    (
-        lxc_set_suite_env
-        for ip in $(global_IPs) ; do
-            if [[ $ip =~ .*:.* ]]; then
-                info_msg "(${ip%|*}) IPv6:       http://[${ip#*|}]"
-            else
-                # IPv4:
-                # shellcheck disable=SC2034,SC2031
-                info_msg "(${ip%|*}) docs-live:  http://${ip#*|}:8080/"
-            fi
-        done
-        "${LXC_REPO_ROOT}/utils/searxng.sh" searxng.instance.env
-    )
-}

utils/lxc.sh

@@ -1,573 +0,0 @@
-#!/usr/bin/env bash
-# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
-# SPDX-License-Identifier: AGPL-3.0-or-later
-
-# shellcheck source=utils/lib.sh
-source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-
-# load environment of the LXC suite
-LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searxng.env}"
-source "$LXC_ENV"
-lxc_set_suite_env
-
-# ----------------------------------------------------------------------------
-# config
-# ----------------------------------------------------------------------------
-#
-# read also:
-# - https://lxd.readthedocs.io/en/latest/
-
-LXC_HOST_PREFIX="${LXC_HOST_PREFIX:-test}"
-
-# Location in the container where all folders from HOST are mounted
-LXC_SHARE_FOLDER="/share"
-LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")"
-
-# shellcheck disable=SC2034
-ubu2004_boilerplate="
-export DEBIAN_FRONTEND=noninteractive
-apt-get update -y
-apt-get upgrade -y
-apt-get install -y git curl wget
-echo 'Set disable_coredump false' >> /etc/sudo.conf
-"
-
-# shellcheck disable=SC2034
-ubu2204_boilerplate="$ubu2004_boilerplate"
-
-# shellcheck disable=SC2034
-archlinux_boilerplate="
-pacman --noprogressbar -Syu --noconfirm
-pacman --noprogressbar -S --noconfirm inetutils git curl wget sudo
-echo 'Set disable_coredump false' >> /etc/sudo.conf
-"
-
-# shellcheck disable=SC2034
-fedora35_boilerplate="
-dnf update -y
-dnf install -y git curl wget hostname
-echo 'Set disable_coredump false' >> /etc/sudo.conf
-"
-
-# shellcheck disable=SC2034
-centos7_boilerplate="
-yum update -y
-yum install -y git curl wget hostname sudo which
-echo 'Set disable_coredump false' >> /etc/sudo.conf
-"
-
-REMOTE_IMAGES=()
-CONTAINERS=()
-LOCAL_IMAGES=()
-
-for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
-    REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${LXC_SUITE[i]}")
-    CONTAINERS=("${CONTAINERS[@]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}")
-    LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${LXC_SUITE[i+1]}")
-done
-
-HOST_USER="${SUDO_USER:-$USER}"
-HOST_USER_ID=$(id -u "${HOST_USER}")
-HOST_GROUP_ID=$(id -g "${HOST_USER}")
-
-# ----------------------------------------------------------------------------
-usage() {
-# ----------------------------------------------------------------------------
-    _cmd="$(basename "$0")"
-    cat <<EOF
-usage::
-  $_cmd build        [containers|<name>]
-  $_cmd copy         [images]
-  $_cmd remove       [containers|<name>|images]
-  $_cmd [start|stop] [containers|<name>]
-  $_cmd show         [images|suite|info|config [<name>]]
-  $_cmd cmd          [--|<name>] '...'
-  $_cmd install      [suite|base [<name>]]
-
-build
-  :containers:   build, launch all containers and 'install base' packages
-  :<name>:       build, launch container <name>  and 'install base' packages
-copy:
-  :images:       copy remote images of the suite into local storage
-remove
-  :containers:   delete all 'containers' or only <container-name>
-  :images:       delete local images of the suite
-start/stop
-  :containers:   start/stop all 'containers' from the suite
-  :<name>:       start/stop container <name> from suite
-show
-  :info:         show info of all (or <name>) containers from LXC suite
-  :config:       show config of all (or <name>) containers from the LXC suite
-  :suite:        show services of all (or <name>) containers from the LXC suite
-  :images:       show information of local images
-cmd
-  use single quotes to evaluate in container's bash, e.g.: 'echo \$(hostname)'
-  --             run command '...' in all containers of the LXC suite
-  :<name>:       run command '...' in container <name>
-install
-  :base:         prepare LXC; install basic packages
-  :suite:        install LXC ${LXC_SUITE_NAME} suite into all (or <name>) containers
-
-EOF
-    usage_containers
-    [ -n "${1+x}" ] &&  err_msg "$1"
-}
-
-usage_containers() {
-    lxc_suite_install_info
-    [ -n "${1+x}" ] &&  err_msg "$1"
-}
-
-lxd_info() {
-
-    cat <<EOF
-
-LXD is needed, to install run::
-
-  snap install lxd
-  lxd init --auto
-
-EOF
-}
-
-main() {
-
-    local exit_val
-    local _usage="unknown or missing $1 command $2"
-
-    lxc_distro_setup
-
-    # don't check prerequisite when in recursion
-    if [[ ! $1 == __* ]] && [[ ! $1 == --help  ]]; then
-        if ! in_container; then
-            ! required_commands lxc && lxd_info && exit 42
-        fi
-        [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42
-    fi
-
-    case $1 in
-        --getenv)  var="$2"; echo "${!var}"; exit 0;;
-        -h|--help) usage; exit 0;;
-
-        build)
-            sudo_or_exit
-            case $2 in
-                ${LXC_HOST_PREFIX}-*) build_container "$2" ;;
-                ''|--|containers) build_all_containers ;;
-                *) usage "$_usage"; exit 42;;
-            esac
-            ;;
-        copy)
-            case $2 in
-                ''|images) lxc_copy_images_locally;;
-                *) usage "$_usage"; exit 42;;
-            esac
-            ;;
-        remove)
-            sudo_or_exit
-            case $2 in
-                ''|--|containers) remove_containers ;;
-                images) lxc_delete_images_locally ;;
-                ${LXC_HOST_PREFIX}-*)
-                    ! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0
-                    if ask_yn "Do you really want to delete container $2"; then
-                        lxc_delete_container "$2"
-                    fi
-                    ;;
-                *) usage "unknown or missing container <name> $2"; exit 42;;
-            esac
-            ;;
-        start|stop)
-            sudo_or_exit
-            case $2 in
-                ''|--|containers)  lxc_cmd "$1" ;;
-                ${LXC_HOST_PREFIX}-*)
-                    ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42
-                    info_msg "lxc $1 $2"
-                    lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] "
-                    ;;
-                *) usage "unknown or missing container <name> $2"; exit 42;;
-            esac
-            ;;
-        show)
-            sudo_or_exit
-            case $2 in
-                suite)
-                    case $3 in
-                        ${LXC_HOST_PREFIX}-*)
-                            lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
-                                | prefix_stdout "[${_BBlue}$3${_creset}]  "
-                        ;;
-                        *) show_suite;;
-                    esac
-                    ;;
-                images) show_images ;;
-                config)
-                    case $3 in
-                        ${LXC_HOST_PREFIX}-*)
-                            ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
-                            lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
-                        ;;
-                        *)
-                            rst_title "container configurations"
-                            echo
-                            lxc list "$LXC_HOST_PREFIX-"
-                            echo
-                            lxc_cmd config show
-                            ;;
-                    esac
-                    ;;
-                info)
-                    case $3 in
-                        ${LXC_HOST_PREFIX}-*)
-                            ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
-                            lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] "
-                            ;;
-                        *)
-                            rst_title "container info"
-                            echo
-                            lxc_cmd info
-                            ;;
-                    esac
-                    ;;
-                *) usage "$_usage"; exit 42;;
-            esac
-            ;;
-        __show)
-            # wrapped show commands, called once in each container
-            case $2 in
-                suite) lxc_suite_info ;;
-            esac
-            ;;
-        cmd)
-            sudo_or_exit
-            shift
-            case $1 in
-                --) shift; lxc_exec "$@" ;;
-                ${LXC_HOST_PREFIX}-*)
-                    ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42
-                    local name=$1
-                    shift
-                    lxc_exec_cmd "${name}" "$@"
-                    ;;
-                *) usage_containers "unknown container: $1" && exit 42
-           esac
-            ;;
-        install)
-            sudo_or_exit
-            case $2 in
-                suite|base)
-                    case $3 in
-                        ${LXC_HOST_PREFIX}-*)
-                            ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42
-                            lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2"
-                            ;;
-                        ''|--) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;;
-                        *) usage_containers "unknown container: $3" && exit 42
-                    esac
-                    ;;
-                *) usage "$_usage"; exit 42 ;;
-            esac
-            ;;
-        __install)
-            # wrapped install commands, called once in each container
-            # shellcheck disable=SC2119
-            case $2 in
-                suite) lxc_suite_install ;;
-                base) FORCE_TIMEOUT=0 lxc_install_base_packages ;;
-            esac
-            ;;
-        doc)
-            echo
-            echo ".. generic utils/lxc.sh documentation"
-            ;;
-        -*) usage "unknown option $1"; exit 42;;
-        *)  usage "unknown or missing command $1"; exit 42;;
-    esac
-}
-
-
-build_all_containers() {
-    rst_title "Build all LXC containers of suite"
-    echo
-    usage_containers
-    lxc_copy_images_locally
-    lxc_init_all_containers
-    lxc_config_all_containers
-    lxc_boilerplate_all_containers
-    rst_title "install LXC base packages" section
-    echo
-    lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base
-    echo
-    lxc list "$LXC_HOST_PREFIX"
-}
-
-build_container() {
-    rst_title "Build container $1"
-
-    local remote_image
-    local container
-    local image
-    local boilerplate_script
-
-    for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
-        if [ "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" = "$1" ]; then
-            remote_image="${LXC_SUITE[i]}"
-            container="${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}"
-            image="${LXC_SUITE[i+1]}"
-            boilerplate_script="${image}_boilerplate"
-            boilerplate_script="${!boilerplate_script}"
-            break
-        fi
-    done
-    echo
-    if [ -z "$container" ]; then
-        err_msg "container $1 unknown"
-        usage_containers
-        return 42
-    fi
-    lxc_image_copy "${remote_image}" "${image}"
-    rst_title "init container" section
-    lxc_init_container "${image}" "${container}"
-    rst_title "configure container" section
-    lxc_config_container "${container}"
-    rst_title "run LXC boilerplate scripts" section
-    lxc_install_boilerplate "${container}" "$boilerplate_script"
-    echo
-    rst_title "install LXC base packages" section
-    lxc_exec_cmd "${container}" "${LXC_REPO_ROOT}/utils/lxc.sh" __install base \
-        | prefix_stdout "[${_BBlue}${container}${_creset}] "
-    echo
-    lxc list "$container"
-}
-
-remove_containers() {
-    rst_title "Remove all LXC containers of suite"
-    rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}"
-    echo
-    lxc list "$LXC_HOST_PREFIX-"
-    echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n  ${CONTAINERS[*]}\\n" | $FMT
-    local default=Ny
-    [[ $FORCE_TIMEOUT = 0 ]] && default=Yn
-    if ask_yn "Do you really want to delete these containers" $default; then
-        for i in "${CONTAINERS[@]}"; do
-            lxc_delete_container "$i"
-        done
-    fi
-    echo
-    lxc list "$LXC_HOST_PREFIX-"
-}
-
-# images
-# ------
-
-lxc_copy_images_locally() {
-    rst_title "copy images" section
-    for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
-        lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}"
-    done
-    # lxc image list local: && wait_key
-}
-
-lxc_delete_images_locally() {
-    rst_title "Delete LXC images"
-    rst_para "local existing images"
-    echo
-    lxc image list local:
-    echo -en "\\n${_BRed}LXC images to delete::${_creset}\\n\\n  ${LOCAL_IMAGES[*]}\\n"
-    if ask_yn "Do you really want to delete these images"; then
-        for i in "${LOCAL_IMAGES[@]}"; do
-            lxc_delete_local_image "$i"
-        done
-    fi
-
-    for i in $(lxc image list --format csv | grep '^,' | sed 's/,\([^,]*\).*$/\1/'); do
-        if ask_yn "Image $i has no alias, do you want to delete the image?" Yn; then
-            lxc_delete_local_image "$i"
-        fi
-    done
-
-    echo
-    lxc image list local:
-}
-
-show_images(){
-    rst_title "local images"
-    echo
-    lxc image list local:
-    echo -en "\\n${_Green}LXC suite images::${_creset}\\n\\n  ${LOCAL_IMAGES[*]}\\n"
-    wait_key
-    for i in "${LOCAL_IMAGES[@]}"; do
-        if lxc_image_exists "$i"; then
-            info_msg "lxc image info ${_BBlue}${i}${_creset}"
-            lxc image info "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
-        else
-            warn_msg "image ${_BBlue}$i${_creset} does not yet exists"
-        fi
-    done
-
-}
-
-
-# container
-# ---------
-
-show_suite(){
-    rst_title "LXC suite ($LXC_HOST_PREFIX-*)"
-    echo
-    lxc list "$LXC_HOST_PREFIX-"
-    echo
-    for i in "${CONTAINERS[@]}"; do
-        if ! lxc_exists "$i"; then
-            warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
-        else
-            lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \
-                | prefix_stdout "[${_BBlue}${i}${_creset}]  "
-            echo
-        fi
-    done
-}
-
-lxc_cmd() {
-    for i in "${CONTAINERS[@]}"; do
-        if ! lxc_exists "$i"; then
-            warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
-        else
-            info_msg "lxc $* $i"
-            lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] "
-        fi
-    done
-}
-
-lxc_exec_cmd() {
-    local name="$1"
-    shift
-    exit_val=
-    info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}"
-    lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*"
-    exit_val=$?
-    if [[ $exit_val -ne 0 ]]; then
-        warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}"
-    else
-        info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}"
-    fi
-}
-
-lxc_exec() {
-    for i in "${CONTAINERS[@]}"; do
-        if ! lxc_exists "$i"; then
-            warn_msg "container ${_BBlue}$i${_creset} does not yet exists"
-        else
-            lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] "
-        fi
-    done
-}
-
-lxc_init_all_containers() {
-    rst_title "init all containers" section
-
-    local image_name
-    local container_name
-
-    for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
-        lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}"
-    done
-}
-
-lxc_config_all_containers() {
-    rst_title "configure all containers" section
-
-    for i in "${CONTAINERS[@]}"; do
-        lxc_config_container "${i}"
-    done
-}
-
-lxc_config_container() {
-    info_msg "[${_BBlue}$1${_creset}] configure container ..."
-
-    info_msg "[${_BBlue}$1${_creset}] map uid/gid from host to container"
-    # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps
-    echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\
-        | lxc config set "$1" raw.idmap -
-
-    info_msg "[${_BBlue}$1${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container"
-    # https://lxd.readthedocs.io/en/latest/instances/#type-disk
-    lxc config device add "$1" repo_share disk \
-        source="${REPO_ROOT}" \
-        path="${LXC_REPO_ROOT}" &>/dev/null
-    # lxc config show "$1" && wait_key
-}
-
-lxc_boilerplate_all_containers() {
-    rst_title "run LXC boilerplate scripts" section
-
-    local boilerplate_script
-    local image_name
-
-    for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do
-
-        image_name="${LXC_SUITE[i+1]}"
-        boilerplate_script="${image_name}_boilerplate"
-        boilerplate_script="${!boilerplate_script}"
-
-        lxc_install_boilerplate "${LXC_HOST_PREFIX}-${image_name}" "$boilerplate_script"
-
-        if [[ -z "${boilerplate_script}" ]]; then
-            err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'"
-        fi
-    done
-}
-
-lxc_install_boilerplate() {
-
-    # usage:  lxc_install_boilerplate <container-name> <string: shell commands ..>
-    #
-    # usage:  lxc_install_boilerplate searx-archlinux "${archlinux_boilerplate}"
-
-    local container_name="$1"
-    local boilerplate_script="$2"
-
-    info_msg "[${_BBlue}${container_name}${_creset}] init .."
-    if lxc start -q "${container_name}" &>/dev/null; then
-        sleep 5 # guest needs some time to come up and get an IP
-    fi
-    if ! check_connectivity "${container_name}"; then
-        die 42 "Container ${container_name} has no internet connectivity!"
-    fi
-    lxc_init_container_env "${container_name}"
-    info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .."
-    cat <<EOF | lxc exec "${container_name}" -- bash | prefix_stdout "[${_BBlue}${container_name}${_creset}] "
-rm -f "/.lxcenv.mk"
-ln -s "${LXC_REPO_ROOT}/utils/makefile.lxc" "/.lxcenv.mk"
-ls -l "/.lxcenv.mk"
-EOF
-
-    info_msg "[${_BBlue}${container_name}${_creset}] run LXC boilerplate scripts .."
-    if lxc start -q "${container_name}" &>/dev/null; then
-        sleep 5 # guest needs some time to come up and get an IP
-    fi
-    if [[ -n "${boilerplate_script}" ]]; then
-        echo "${boilerplate_script}" \
-            | lxc exec "${container_name}" -- bash \
-            | prefix_stdout "[${_BBlue}${container_name}${_creset}] "
-    fi
-}
-
-check_connectivity() {
-    local ret_val=0
-    info_msg "check internet connectivity ..."
-    if ! lxc exec "${1}" -- ping -c 1 9.9.9.9 &>/dev/null; then
-        ret_val=1
-        err_msg "no internet connectivity!"
-        info_msg "Most often the connectivity is blocked by a docker installation:"
-        info_msg "Whenever docker is started (reboot) it sets the iptables policy "
-        info_msg "for the FORWARD chain to DROP, see:"
-        info_msg "    https://docs.searxng.org/utils/lxc.sh.html#internet-connectivity-docker"
-        iptables-save | grep ":FORWARD"
-    fi
-    return $ret_val
-}
-
-# ----------------------------------------------------------------------------
-main "$@"
-# ----------------------------------------------------------------------------

utils/makefile.include

@@ -1,22 +1,8 @@
 # -*- coding: utf-8; mode: makefile-gmake -*-
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-ifeq (,$(wildcard /.lxcenv.mk))
-PHONY += lxc-activate lxc-purge
-lxc-activate:
-	@$(MAKE) -s -f "$$(dirname $(abspath $(lastword $(MAKEFILE_LIST))))/makefile.lxc" lxc-activate
-lxc-purge:
-	$(Q)rm -rf ./lxc-env
-else
-	include /.lxcenv.mk
-endif
-
 PHONY += make-help
-ifeq (,$(wildcard /.lxcenv.mk))
 make-help:
-else
-make-help: lxc-help
-endif
 	@echo  'options:'
 	@echo  '  make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'
 	@echo  '  make V=2   [targets] 2 => give reason for rebuild of target'

utils/makefile.lxc

@@ -1,32 +0,0 @@
-# -*- coding: utf-8; mode: makefile-gmake -*-
-#
-# LXC environment
-# ===============
-#
-# To activate/deactivate LXC makefile environment in a container, set/unset link
-# from root '/.lxcenv.mk' to *this* file::
-#
-#   sudo make ./utils/makefile.lxc lxc-activate
-#   sudo make ./utils/makefile.lxc lxc-deactivate
-
-LXC_ENV_FOLDER=lxc-env/$(shell hostname)/
-
-lxc-help::
-	@echo  '  LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)'
-
-# If not activated, serve target 'lxc-activate' ..
-ifeq (,$(wildcard /.lxcenv.mk))
-PHONY += lxc-activate
-lxc-activate:
-	ln -s "$(abspath $(lastword $(MAKEFILE_LIST)))" "/.lxcenv.mk"
-else
-# .. and if activated, serve target 'lxc-deactivate'.
-PHONY += lxc-deactivate
-lxc-deactivate:
-	rm /.lxcenv.mk
-$(LXC_ENV_FOLDER):
-	$(Q)mkdir -p $(LXC_ENV_FOLDER)
-	$(Q)echo placeholder > $(LXC_ENV_FOLDER).placeholder
-endif
-
-.PHONY: $(PHONY)

utils/searxng.sh

@@ -41,10 +41,6 @@ fi
 
 SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)/searxng}"
 SEARXNG_URL="${SEARXNG_URL%/}" # if exists, remove trailing slash
-if in_container; then
-    # hint: Linux containers do not have DNS entries, lets use IPs
-    SEARXNG_URL="http://$(primary_ip)/searxng"
-fi
 SEARXNG_URL_PATH="$(echo "${SEARXNG_URL}" | sed -e 's,^.*://[^/]*\(/.*\),\1,g')"
 [[ "${SEARXNG_URL_PATH}" == "${SEARXNG_URL}" ]] && SEARXNG_URL_PATH=/
 
@@ -114,9 +110,7 @@ esac
 
 _service_prefix="  ${_Yellow}|${SERVICE_USER}|${_creset} "
 
-# ----------------------------------------------------------------------------
 usage() {
-# ----------------------------------------------------------------------------
 
     # shellcheck disable=SC1117
     cat <<EOF
@@ -175,17 +169,27 @@ main() {
             apache_distro_setup
             uWSGI_distro_setup
             required_commands \
-                sudo systemctl install git wget curl \
+                sudo systemctl install git wget curl ||
-                || exit
+                exit
             ;;
     esac
 
     local _usage="unknown or missing $1 command $2"
 
     case $1 in
-        --getenv)  var="$2"; echo "${!var}"; exit 0;;
+        --getenv)
-        --cmd)  shift; "$@";;
+            var="$2"
-        -h|--help) usage; exit 0;;
+            echo "${!var}"
+            exit 0
+            ;;
+        --cmd)
+            shift
+            "$@"
+            ;;
+        -h | --help)
+            usage
+            exit 0
+            ;;
         install)
             sudo_or_exit
             case $2 in
@@ -200,7 +204,10 @@ main() {
                 nginx) searxng.nginx.install ;;
                 apache) searxng.apache.install ;;
                 valkey) searxng.install.valkey ;;
-                *) usage "$_usage"; exit 42;;
+                *)
+                    usage "$_usage"
+                    exit 42
+                    ;;
             esac
             ;;
         remove)
@@ -215,7 +222,10 @@ main() {
                 remove) searxng.nginx.remove ;;
                 valkey) searxng.remove.valkey ;;
                 redis) searxng.remove.redis ;;
-                *) usage "$_usage"; exit 42;;
+                *)
+                    usage "$_usage"
+                    exit 42
+                    ;;
             esac
             ;;
         instance)
@@ -234,19 +244,30 @@ main() {
                     ;;
                 cmd)
                     sudo_or_exit
-                    shift; shift; searxng.instance.exec "$@"
+                    shift
+                    shift
+                    searxng.instance.exec "$@"
                     ;;
                 get_setting)
-                    shift; shift; searxng.instance.get_setting "$@"
+                    shift
+                    shift
+                    searxng.instance.get_setting "$@"
                     ;;
                 call)
                     # call a function in instance's environment
-                    shift; shift; searxng.instance.self.call "$@"
+                    shift
+                    shift
+                    searxng.instance.self.call "$@"
                     ;;
                 _call)
-                    shift; shift; "$@"
+                    shift
+                    shift
+                    "$@"
+                    ;;
+                *)
+                    usage "$_usage"
+                    exit 42
                     ;;
-                *) usage "$_usage"; exit 42;;
             esac
             ;;
         *)
@@ -617,7 +638,6 @@ searxng.install.valkey() {
     valkey.install
 }
 
-
 searxng.instance.localtest() {
     rst_title "Test SearXNG instance locally" section
     rst_para "Activate debug mode, start a minimal SearXNG " \
@@ -840,12 +860,6 @@ searxng.instance.inspect() {
 _searxng.instance.inspect() {
     searxng.instance.env
 
-    if in_container; then
-        # shellcheck source=utils/lxc-searxng.env
-        source "${REPO_ROOT}/utils/lxc-searxng.env"
-        lxc_suite_info
-    fi
-
     MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue${_creset}"
 
     if ! searxng.uwsgi.available; then
@@ -897,7 +911,8 @@ searxng.doc.rst() {
             echo -e "\n.. START searxng uwsgi-description $DIST_NAME"
 
             case $DIST_ID-$DIST_VERS in
-                ubuntu-*|debian-*)  cat <<EOF
+                ubuntu-* | debian-*)
+                    cat <<EOF
 
 .. code:: bash
 
@@ -914,7 +929,8 @@ searxng.doc.rst() {
 
 EOF
                     ;;
-                arch-*) cat <<EOF
+                arch-*)
+                    cat <<EOF
 
 .. code:: bash
 
@@ -932,7 +948,8 @@ EOF
 
 EOF
                     ;;
-                fedora-*|centos-7) cat <<EOF
+                fedora-* | centos-7)
+                    cat <<EOF
 
 .. code:: bash