Compare commits

...

5 Commits

Author SHA1 Message Date
Markus Heiser 53e85309f9
Merge 1c9b28968d into cd384a8a60 2024-11-07 18:18:41 +08:00
dependabot[bot] cd384a8a60 [upd] pypi: Bump selenium from 4.25.0 to 4.26.1
Bumps [selenium](https://github.com/SeleniumHQ/Selenium) from 4.25.0 to 4.26.1.
- [Release notes](https://github.com/SeleniumHQ/Selenium/releases)
- [Commits](https://github.com/SeleniumHQ/Selenium/commits)

---
updated-dependencies:
- dependency-name: selenium
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-11-06 10:01:13 +01:00
Markus Heiser c4055e449f [fix] issues reported by `make test.yamllint`
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2024-11-06 08:16:21 +01:00
Markus Heiser 2fdbf2622b [mod] lint github YAML config files
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2024-11-06 08:16:21 +01:00
Markus Heiser 1c9b28968d [mod] botdetection: HTTP Fetch Metadata Request Headers
HTTP Fetch Metadata Request Headers [1][2] are used to detect bot requests. Bots
with invalid *Fetch Metadata* will be redirected to the intro (`index`)  page.

[1] https://www.w3.org/TR/fetch-metadata/
[2] https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2024-10-27 13:42:57 +01:00
11 changed files with 242 additions and 172 deletions

View File

@ -1,5 +1,5 @@
name: "Checker" name: "Checker"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "0 4 * * 5" - cron: "0 4 * * 5"
workflow_dispatch: workflow_dispatch:

View File

@ -1,5 +1,5 @@
name: "Update searx.data" name: "Update searx.data"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "59 23 28 * *" - cron: "59 23 28 * *"
workflow_dispatch: workflow_dispatch:

View File

@ -1,6 +1,6 @@
name: Integration name: Integration
on: on: # yamllint disable-line rule:truthy
push: push:
branches: ["master"] branches: ["master"]
pull_request: pull_request:
@ -16,62 +16,62 @@ jobs:
strategy: strategy:
matrix: matrix:
os: [ubuntu-20.04] os: [ubuntu-20.04]
python-version: ["3.9", "3.10", "3.11", "3.12",] python-version: ["3.9", "3.10", "3.11", "3.12"]
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: | run: |
sudo ./utils/searxng.sh install packages sudo ./utils/searxng.sh install packages
sudo apt install firefox sudo apt install firefox
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: ${{ matrix.python-version }} python-version: ${{ matrix.python-version }}
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }} key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: Install Python dependencies - name: Install Python dependencies
if: steps.cache-python.outputs.cache-hit != 'true' if: steps.cache-python.outputs.cache-hit != 'true'
run: | run: |
make V=1 install make V=1 install
make V=1 gecko.driver make V=1 gecko.driver
- name: Run tests - name: Run tests
run: make V=1 ci.test run: make V=1 ci.test
themes: themes:
name: Themes name: Themes
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: '3.12'
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: Install node dependencies - name: Install node dependencies
run: make V=1 node.env run: make V=1 node.env
- name: Build themes - name: Build themes
run: make V=1 themes.all run: make V=1 themes.all
documentation: documentation:
name: Documentation name: Documentation
@ -79,40 +79,40 @@ jobs:
permissions: permissions:
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo contents: write # for JamesIves/github-pages-deploy-action to push changes in repo
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
fetch-depth: '0' fetch-depth: '0'
persist-credentials: false persist-credentials: false
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: '3.12'
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: Build documentation - name: Build documentation
run: | run: |
make V=1 docs.clean docs.html make V=1 docs.clean docs.html
- name: Deploy - name: Deploy
if: github.ref == 'refs/heads/master' if: github.ref == 'refs/heads/master'
uses: JamesIves/github-pages-deploy-action@3.7.1 uses: JamesIves/github-pages-deploy-action@3.7.1
with: with:
GITHUB_TOKEN: ${{ github.token }} GITHUB_TOKEN: ${{ github.token }}
BRANCH: gh-pages BRANCH: gh-pages
FOLDER: dist/docs FOLDER: dist/docs
CLEAN: true # Automatically remove deleted files from the deploy branch CLEAN: true # Automatically remove deleted files from the deploy branch
SINGLE_COMMIT: True SINGLE_COMMIT: true
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}' COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
babel: babel:
name: Update translations branch name: Update translations branch
@ -125,37 +125,37 @@ jobs:
permissions: permissions:
contents: write # for make V=1 weblate.push.translations contents: write # for make V=1 weblate.push.translations
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
fetch-depth: '0' fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: '3.12'
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: weblate & git setup - name: weblate & git setup
env: env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
run: | run: |
mkdir -p ~/.config mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com" git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot" git config --global user.name "searxng-bot"
- name: Update transations - name: Update transations
id: update id: update
run: | run: |
make V=1 weblate.push.translations make V=1 weblate.push.translations
dockers: dockers:
name: Docker name: Docker

View File

@ -1,5 +1,5 @@
name: "Security checks" name: "Security checks"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "42 05 * * *" - cron: "42 05 * * *"
workflow_dispatch: workflow_dispatch:

View File

@ -1,5 +1,5 @@
name: "Update translations" name: "Update translations"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "05 07 * * 5" - cron: "05 07 * * 5"
workflow_dispatch: workflow_dispatch:
@ -10,50 +10,50 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
fetch-depth: '0' fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: '3.12'
architecture: 'x64' architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: weblate & git setup - name: weblate & git setup
env: env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
run: | run: |
mkdir -p ~/.config mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com" git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot" git config --global user.name "searxng-bot"
- name: Merge and push transation updates - name: Merge and push transation updates
run: | run: |
make V=1 weblate.translations.commit make V=1 weblate.translations.commit
- name: Create Pull Request - name: Create Pull Request
id: cpr id: cpr
uses: peter-evans/create-pull-request@v3 uses: peter-evans/create-pull-request@v3
with: with:
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
commit-message: '[l10n] update translations from Weblate' commit-message: '[l10n] update translations from Weblate'
committer: searxng-bot <searxng-bot@users.noreply.github.com> committer: searxng-bot <searxng-bot@users.noreply.github.com>
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
signoff: false signoff: false
branch: translations_update branch: translations_update
delete-branch: true delete-branch: true
draft: false draft: false
title: '[l10n] update translations from Weblate' title: '[l10n] update translations from Weblate'
body: | body: |
update translations from Weblate update translations from Weblate
labels: | labels: |
translation translation

View File

@ -53,6 +53,9 @@ Probe HTTP headers
.. automodule:: searx.botdetection.http_user_agent .. automodule:: searx.botdetection.http_user_agent
:members: :members:
.. automodule:: searx.botdetection.sec_fetch
:members:
.. _botdetection config: .. _botdetection config:
Config Config

2
manage
View File

@ -57,7 +57,7 @@ while IFS= read -r line; do
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
YAMLLINT_FILES+=("$line") YAMLLINT_FILES+=("$line")
fi fi
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')" done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
RST_FILES=( RST_FILES=(
'README.rst' 'README.rst'

View File

@ -4,7 +4,7 @@ cov-core==1.15.0
black==24.3.0 black==24.3.0
pylint==3.3.1 pylint==3.3.1
splinter==0.21.0 splinter==0.21.0
selenium==4.25.0 selenium==4.26.1
Pallets-Sphinx-Themes==2.3.0 Pallets-Sphinx-Themes==2.3.0
Sphinx==7.4.7 Sphinx==7.4.7
sphinx-issues==5.0.0 sphinx-issues==5.0.0

View File

@ -31,6 +31,9 @@ def dump_request(request: flask.Request):
+ " || Content-Length: %s" % request.headers.get('Content-Length') + " || Content-Length: %s" % request.headers.get('Content-Length')
+ " || Connection: %s" % request.headers.get('Connection') + " || Connection: %s" % request.headers.get('Connection')
+ " || User-Agent: %s" % request.headers.get('User-Agent') + " || User-Agent: %s" % request.headers.get('User-Agent')
+ " || Sec-Fetch-Site: %s" % request.headers.get('Sec-Fetch-Site')
+ " || Sec-Fetch-Mode: %s" % request.headers.get('Sec-Fetch-Mode')
+ " || Sec-Fetch-Dest: %s" % request.headers.get('Sec-Fetch-Dest')
) )

View File

@ -0,0 +1,59 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""
Method ``http_sec_fetch``
-------------------------
The ``http_sec_fetch`` method protect resources from web attacks with `Fetch
Metadata`_. A request is filtered out in case of:
- http header Sec-Fetch-Mode_ is invalid
- http header Sec-Fetch-Dest_ is invalid
.. _Fetch Metadata:
https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header
.. Sec-Fetch-Dest:
https://developer.mozilla.org/en-US/docs/Web/API/Request/destination
.. Sec-Fetch-Mode:
https://developer.mozilla.org/en-US/docs/Web/API/Request/mode
"""
# pylint: disable=unused-argument
from __future__ import annotations
from ipaddress import (
IPv4Network,
IPv6Network,
)
import flask
import werkzeug
from . import config
from ._helpers import logger
def filter_request(
network: IPv4Network | IPv6Network,
request: flask.Request,
cfg: config.Config,
) -> werkzeug.Response | None:
val = request.headers.get("Sec-Fetch-Mode", "")
if val != "navigate":
logger.debug("invalid Sec-Fetch-Mode '%s'", val)
return flask.redirect(flask.url_for('index'), code=302)
val = request.headers.get("Sec-Fetch-Site", "")
if val not in ('same-origin', 'same-site', 'none'):
logger.debug("invalid Sec-Fetch-Site '%s'", val)
flask.redirect(flask.url_for('index'), code=302)
val = request.headers.get("Sec-Fetch-Dest", "")
if val != "document":
logger.debug("invalid Sec-Fetch-Dest '%s'", val)
flask.redirect(flask.url_for('index'), code=302)
return None

View File

@ -111,6 +111,7 @@ from searx.botdetection import (
http_accept_encoding, http_accept_encoding,
http_accept_language, http_accept_language,
http_user_agent, http_user_agent,
http_sec_fetch,
ip_limit, ip_limit,
ip_lists, ip_lists,
get_network, get_network,
@ -178,16 +179,17 @@ def filter_request(request: flask.Request) -> werkzeug.Response | None:
logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg) logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg)
return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429)) return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429))
# methods applied on / # methods applied on all requests
for func in [ for func in [
http_user_agent, http_user_agent,
]: ]:
val = func.filter_request(network, request, cfg) val = func.filter_request(network, request, cfg)
if val is not None: if val is not None:
logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(flask.request))
return val return val
# methods applied on /search # methods applied on /search requests
if request.path == '/search': if request.path == '/search':
@ -196,12 +198,15 @@ def filter_request(request: flask.Request) -> werkzeug.Response | None:
http_accept_encoding, http_accept_encoding,
http_accept_language, http_accept_language,
http_user_agent, http_user_agent,
http_sec_fetch,
ip_limit, ip_limit,
]: ]:
val = func.filter_request(network, request, cfg) val = func.filter_request(network, request, cfg)
if val is not None: if val is not None:
logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(flask.request))
return val return val
logger.debug(f"OK {network}: %s", dump_request(flask.request)) logger.debug(f"OK: {network}: %s", dump_request(flask.request))
return None return None