Compare commits

...

5 Commits

Author SHA1 Message Date
Markus Heiser 4f5bb1bdf7
Merge e28a69515c into cd384a8a60 2024-11-06 10:03:21 +01:00
dependabot[bot] cd384a8a60 [upd] pypi: Bump selenium from 4.25.0 to 4.26.1
Bumps [selenium](https://github.com/SeleniumHQ/Selenium) from 4.25.0 to 4.26.1.
- [Release notes](https://github.com/SeleniumHQ/Selenium/releases)
- [Commits](https://github.com/SeleniumHQ/Selenium/commits)

---
updated-dependencies:
- dependency-name: selenium
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-11-06 10:01:13 +01:00
Markus Heiser c4055e449f [fix] issues reported by `make test.yamllint`
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2024-11-06 08:16:21 +01:00
Markus Heiser 2fdbf2622b [mod] lint github YAML config files
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2024-11-06 08:16:21 +01:00
Markus Heiser e28a69515c [mod] uWSGI config: configuring uwsgi for production
As stated in .. and other posts, the defaults of uWSGI not suitable for a
productive environment.  To give just one example, the workers run indefinitely
and the memory leaks aggregate.

- "Configuring uWSGI for Production: The defaults are all wrong" EuroPython 2019 [1]
- "Configuring uWSGI for Production Deployment" [2]
- "When Paul has tested some PR on his instance, we could clearly see a memory
  leak over a week: the memory never dropped to the initial value. Same for my
  instance using Docker." [3]

[1] https://av.tib.eu/media/44810
[2] https://www.bloomberg.com/company/stories/configuring-uwsgi-production-deployment/
[3] https://github.com/searxng/searxng/pull/3443#issuecomment-2094347004

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2024-06-23 12:05:15 +02:00
12 changed files with 234 additions and 174 deletions

View File

@ -1,5 +1,5 @@
name: "Checker" name: "Checker"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "0 4 * * 5" - cron: "0 4 * * 5"
workflow_dispatch: workflow_dispatch:

View File

@ -1,5 +1,5 @@
name: "Update searx.data" name: "Update searx.data"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "59 23 28 * *" - cron: "59 23 28 * *"
workflow_dispatch: workflow_dispatch:

View File

@ -1,6 +1,6 @@
name: Integration name: Integration
on: on: # yamllint disable-line rule:truthy
push: push:
branches: ["master"] branches: ["master"]
pull_request: pull_request:
@ -16,62 +16,62 @@ jobs:
strategy: strategy:
matrix: matrix:
os: [ubuntu-20.04] os: [ubuntu-20.04]
python-version: ["3.9", "3.10", "3.11", "3.12",] python-version: ["3.9", "3.10", "3.11", "3.12"]
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: | run: |
sudo ./utils/searxng.sh install packages sudo ./utils/searxng.sh install packages
sudo apt install firefox sudo apt install firefox
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: ${{ matrix.python-version }} python-version: ${{ matrix.python-version }}
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }} key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: Install Python dependencies - name: Install Python dependencies
if: steps.cache-python.outputs.cache-hit != 'true' if: steps.cache-python.outputs.cache-hit != 'true'
run: | run: |
make V=1 install make V=1 install
make V=1 gecko.driver make V=1 gecko.driver
- name: Run tests - name: Run tests
run: make V=1 ci.test run: make V=1 ci.test
themes: themes:
name: Themes name: Themes
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: '3.12'
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: Install node dependencies - name: Install node dependencies
run: make V=1 node.env run: make V=1 node.env
- name: Build themes - name: Build themes
run: make V=1 themes.all run: make V=1 themes.all
documentation: documentation:
name: Documentation name: Documentation
@ -79,40 +79,40 @@ jobs:
permissions: permissions:
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo contents: write # for JamesIves/github-pages-deploy-action to push changes in repo
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
fetch-depth: '0' fetch-depth: '0'
persist-credentials: false persist-credentials: false
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: '3.12'
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: Build documentation - name: Build documentation
run: | run: |
make V=1 docs.clean docs.html make V=1 docs.clean docs.html
- name: Deploy - name: Deploy
if: github.ref == 'refs/heads/master' if: github.ref == 'refs/heads/master'
uses: JamesIves/github-pages-deploy-action@3.7.1 uses: JamesIves/github-pages-deploy-action@3.7.1
with: with:
GITHUB_TOKEN: ${{ github.token }} GITHUB_TOKEN: ${{ github.token }}
BRANCH: gh-pages BRANCH: gh-pages
FOLDER: dist/docs FOLDER: dist/docs
CLEAN: true # Automatically remove deleted files from the deploy branch CLEAN: true # Automatically remove deleted files from the deploy branch
SINGLE_COMMIT: True SINGLE_COMMIT: true
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}' COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
babel: babel:
name: Update translations branch name: Update translations branch
@ -125,37 +125,37 @@ jobs:
permissions: permissions:
contents: write # for make V=1 weblate.push.translations contents: write # for make V=1 weblate.push.translations
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
fetch-depth: '0' fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: '3.12'
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: weblate & git setup - name: weblate & git setup
env: env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
run: | run: |
mkdir -p ~/.config mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com" git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot" git config --global user.name "searxng-bot"
- name: Update transations - name: Update transations
id: update id: update
run: | run: |
make V=1 weblate.push.translations make V=1 weblate.push.translations
dockers: dockers:
name: Docker name: Docker

View File

@ -1,5 +1,5 @@
name: "Security checks" name: "Security checks"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "42 05 * * *" - cron: "42 05 * * *"
workflow_dispatch: workflow_dispatch:

View File

@ -1,5 +1,5 @@
name: "Update translations" name: "Update translations"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "05 07 * * 5" - cron: "05 07 * * 5"
workflow_dispatch: workflow_dispatch:
@ -10,50 +10,50 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
fetch-depth: '0' fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: '3.12'
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: weblate & git setup - name: weblate & git setup
env: env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
run: | run: |
mkdir -p ~/.config mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com" git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot" git config --global user.name "searxng-bot"
- name: Merge and push transation updates - name: Merge and push transation updates
run: | run: |
make V=1 weblate.translations.commit make V=1 weblate.translations.commit
- name: Create Pull Request - name: Create Pull Request
id: cpr id: cpr
uses: peter-evans/create-pull-request@v3 uses: peter-evans/create-pull-request@v3
with: with:
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
commit-message: '[l10n] update translations from Weblate' commit-message: '[l10n] update translations from Weblate'
committer: searxng-bot <searxng-bot@users.noreply.github.com> committer: searxng-bot <searxng-bot@users.noreply.github.com>
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
signoff: false signoff: false
branch: translations_update branch: translations_update
delete-branch: true delete-branch: true
draft: false draft: false
title: '[l10n] update translations from Weblate' title: '[l10n] update translations from Weblate'
body: | body: |
update translations from Weblate update translations from Weblate
labels: | labels: |
translation translation

View File

@ -1,3 +1,4 @@
# -*- mode: conf-unix; coding: utf-8 -*-
[uwsgi] [uwsgi]
# Who will run the code # Who will run the code
uid = searxng uid = searxng
@ -6,6 +7,13 @@ gid = searxng
# Number of workers (usually CPU count) # Number of workers (usually CPU count)
# default value: %k (= number of CPU core, see Dockerfile) # default value: %k (= number of CPU core, see Dockerfile)
workers = $(UWSGI_WORKERS) workers = $(UWSGI_WORKERS)
harakiri = 60
# max-requests = 1000 # Restart workers after this many requests
# max-worker-lifetime = 3600 # Restart workers after this many seconds
reload-on-rss = 4096 # Restart workers after this much resident memory
worker-reload-mercy = 60 # How long to wait before forcefully killing workers
die-on-term = true # Shutdown when receiving SIGTERM (default is respawn)
py-callos-afterfork = true # allow workers to trap signals
# Number of threads per worker # Number of threads per worker
# default value: 4 (see Dockerfile) # default value: 4 (see Dockerfile)
@ -17,6 +25,9 @@ chmod-socket = 666
# Plugin to use and interpreter config # Plugin to use and interpreter config
single-interpreter = true single-interpreter = true
master = true master = true
strict = true
vacuum = true # Delete sockets during shutdown
need-app = true
plugin = python3 plugin = python3
lazy-apps = true lazy-apps = true
enable-threads = true enable-threads = true
@ -33,7 +44,8 @@ auto-procname = true
# Disable request logging for privacy # Disable request logging for privacy
disable-logging = true disable-logging = true
log-5xx = true log-4xx = true # but log 4xx's anyway
log-5xx = true # and 5xx's
# Set the max size of a request (request-body excluded) # Set the max size of a request (request-body excluded)
buffer-size = 8192 buffer-size = 8192

2
manage
View File

@ -57,7 +57,7 @@ while IFS= read -r line; do
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
YAMLLINT_FILES+=("$line") YAMLLINT_FILES+=("$line")
fi fi
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')" done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
RST_FILES=( RST_FILES=(
'README.rst' 'README.rst'

View File

@ -4,7 +4,7 @@ cov-core==1.15.0
black==24.3.0 black==24.3.0
pylint==3.3.1 pylint==3.3.1
splinter==0.21.0 splinter==0.21.0
selenium==4.25.0 selenium==4.26.1
Pallets-Sphinx-Themes==2.3.0 Pallets-Sphinx-Themes==2.3.0
Sphinx==7.4.7 Sphinx==7.4.7
sphinx-issues==5.0.0 sphinx-issues==5.0.0

View File

@ -1,4 +1,4 @@
# -*- mode: conf; coding: utf-8 -*- # -*- mode: conf-unix; coding: utf-8 -*-
[uwsgi] [uwsgi]
# uWSGI core # uWSGI core
@ -24,6 +24,8 @@ env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
# disable logging for privacy # disable logging for privacy
logger = systemd logger = systemd
disable-logging = true disable-logging = true
log-4xx = true # but log 4xx's anyway
log-5xx = true # and 5xx's
# The right granted on the created socket # The right granted on the created socket
chmod-socket = 666 chmod-socket = 666
@ -33,6 +35,9 @@ single-interpreter = true
# enable master process # enable master process
master = true master = true
strict = true
vacuum = true # Delete sockets during shutdown
need-app = true
# load apps in each worker instead of the master # load apps in each worker instead of the master
lazy-apps = true lazy-apps = true
@ -50,6 +55,13 @@ enable-threads = true
# Number of workers (usually CPU count) # Number of workers (usually CPU count)
workers = ${UWSGI_WORKERS:-%k} workers = ${UWSGI_WORKERS:-%k}
threads = ${UWSGI_THREADS:-4} threads = ${UWSGI_THREADS:-4}
harakiri = 60
# max-requests = 1000 # Restart workers after this many requests
# max-worker-lifetime = 3600 # Restart workers after this many seconds
reload-on-rss = 4096 # Restart workers after this much resident memory
worker-reload-mercy = 60 # How long to wait before forcefully killing workers
die-on-term = true # Shutdown when receiving SIGTERM (default is respawn)
py-callos-afterfork = true # allow workers to trap signals
# plugin: python # plugin: python
# -------------- # --------------

View File

@ -1,4 +1,4 @@
# -*- mode: conf; coding: utf-8 -*- # -*- mode: conf-unix; coding: utf-8 -*-
[uwsgi] [uwsgi]
# uWSGI core # uWSGI core
@ -24,6 +24,8 @@ env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
# disable logging for privacy # disable logging for privacy
logger = systemd logger = systemd
disable-logging = true disable-logging = true
log-4xx = true # but log 4xx's anyway
log-5xx = true # and 5xx's
# The right granted on the created socket # The right granted on the created socket
chmod-socket = 666 chmod-socket = 666
@ -33,6 +35,9 @@ single-interpreter = true
# enable master process # enable master process
master = true master = true
strict = true
vacuum = true # Delete sockets during shutdown
need-app = true
# load apps in each worker instead of the master # load apps in each worker instead of the master
lazy-apps = true lazy-apps = true
@ -50,6 +55,13 @@ enable-threads = true
# Number of workers (usually CPU count) # Number of workers (usually CPU count)
workers = ${UWSGI_WORKERS:-%k} workers = ${UWSGI_WORKERS:-%k}
threads = ${UWSGI_THREADS:-4} threads = ${UWSGI_THREADS:-4}
harakiri = 60
# max-requests = 1000 # Restart workers after this many requests
# max-worker-lifetime = 3600 # Restart workers after this many seconds
reload-on-rss = 4096 # Restart workers after this much resident memory
worker-reload-mercy = 60 # How long to wait before forcefully killing workers
die-on-term = true # Shutdown when receiving SIGTERM (default is respawn)
py-callos-afterfork = true # allow workers to trap signals
# plugin: python # plugin: python
# -------------- # --------------

View File

@ -1,4 +1,4 @@
# -*- mode: conf; coding: utf-8 -*- # -*- mode: conf-unix; coding: utf-8 -*-
[uwsgi] [uwsgi]
# uWSGI core # uWSGI core
@ -27,6 +27,8 @@ env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
# disable logging for privacy # disable logging for privacy
disable-logging = true disable-logging = true
log-4xx = true # but log 4xx's anyway
log-5xx = true # and 5xx's
# The right granted on the created socket # The right granted on the created socket
chmod-socket = 666 chmod-socket = 666
@ -36,6 +38,9 @@ single-interpreter = true
# enable master process # enable master process
master = true master = true
strict = true
vacuum = true # Delete sockets during shutdown
need-app = true
# load apps in each worker instead of the master # load apps in each worker instead of the master
lazy-apps = true lazy-apps = true
@ -53,6 +58,13 @@ enable-threads = true
# Number of workers (usually CPU count) # Number of workers (usually CPU count)
workers = ${UWSGI_WORKERS:-%k} workers = ${UWSGI_WORKERS:-%k}
threads = ${UWSGI_THREADS:-4} threads = ${UWSGI_THREADS:-4}
harakiri = 60
# max-requests = 1000 # Restart workers after this many requests
# max-worker-lifetime = 3600 # Restart workers after this many seconds
reload-on-rss = 4096 # Restart workers after this much resident memory
worker-reload-mercy = 60 # How long to wait before forcefully killing workers
die-on-term = true # Shutdown when receiving SIGTERM (default is respawn)
py-callos-afterfork = true # allow workers to trap signals
# plugin: python # plugin: python
# -------------- # --------------

View File

@ -1,4 +1,4 @@
# -*- mode: conf; coding: utf-8 -*- # -*- mode: conf-unix; coding: utf-8 -*-
[uwsgi] [uwsgi]
# uWSGI core # uWSGI core
@ -27,6 +27,8 @@ env = SEARXNG_SETTINGS_PATH=${SEARXNG_SETTINGS_PATH}
# disable logging for privacy # disable logging for privacy
disable-logging = true disable-logging = true
log-4xx = true # but log 4xx's anyway
log-5xx = true # and 5xx's
# The right granted on the created socket # The right granted on the created socket
chmod-socket = 666 chmod-socket = 666
@ -36,6 +38,9 @@ single-interpreter = true
# enable master process # enable master process
master = true master = true
strict = true
vacuum = true # Delete sockets during shutdown
need-app = true
# load apps in each worker instead of the master # load apps in each worker instead of the master
lazy-apps = true lazy-apps = true
@ -53,6 +58,13 @@ enable-threads = true
# Number of workers (usually CPU count) # Number of workers (usually CPU count)
workers = ${UWSGI_WORKERS:-%k} workers = ${UWSGI_WORKERS:-%k}
threads = ${UWSGI_THREADS:-4} threads = ${UWSGI_THREADS:-4}
harakiri = 60
# max-requests = 1000 # Restart workers after this many requests
# max-worker-lifetime = 3600 # Restart workers after this many seconds
reload-on-rss = 4096 # Restart workers after this much resident memory
worker-reload-mercy = 60 # How long to wait before forcefully killing workers
die-on-term = true # Shutdown when receiving SIGTERM (default is respawn)
py-callos-afterfork = true # allow workers to trap signals
# plugin: python # plugin: python
# -------------- # --------------