mirror of https://github.com/searxng/searxng.git
Compare commits
5 Commits
9b1cbb3218
...
aa9c5b4d9b
| Author | SHA1 | Date |
|---|---|---|
Alexandre Flament
|
aa9c5b4d9b | |
dependabot[bot]
|
cd384a8a60 | |
Markus Heiser
|
c4055e449f | |
|
Markus Heiser
|
2fdbf2622b | |
Alexandre FLAMENT
|
f49d1a9b90 |
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Checker"
|
name: "Checker"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "0 4 * * 5"
|
- cron: "0 4 * * 5"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Update searx.data"
|
name: "Update searx.data"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "59 23 28 * *"
|
- cron: "59 23 28 * *"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
name: Integration
|
name: Integration
|
||||||
|
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
push:
|
push:
|
||||||
branches: ["master"]
|
branches: ["master"]
|
||||||
pull_request:
|
pull_request:
|
||||||
|
|
@ -16,62 +16,62 @@ jobs:
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
os: [ubuntu-20.04]
|
os: [ubuntu-20.04]
|
||||||
python-version: ["3.9", "3.10", "3.11", "3.12",]
|
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
- name: Install Ubuntu packages
|
- name: Install Ubuntu packages
|
||||||
run: |
|
run: |
|
||||||
sudo ./utils/searxng.sh install packages
|
sudo ./utils/searxng.sh install packages
|
||||||
sudo apt install firefox
|
sudo apt install firefox
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }}
|
key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }}
|
||||||
- name: Install Python dependencies
|
- name: Install Python dependencies
|
||||||
if: steps.cache-python.outputs.cache-hit != 'true'
|
if: steps.cache-python.outputs.cache-hit != 'true'
|
||||||
run: |
|
run: |
|
||||||
make V=1 install
|
make V=1 install
|
||||||
make V=1 gecko.driver
|
make V=1 gecko.driver
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
run: make V=1 ci.test
|
run: make V=1 ci.test
|
||||||
|
|
||||||
themes:
|
themes:
|
||||||
name: Themes
|
name: Themes
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
- name: Install Ubuntu packages
|
- name: Install Ubuntu packages
|
||||||
run: sudo ./utils/searxng.sh install buildhost
|
run: sudo ./utils/searxng.sh install buildhost
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||||
- name: Install node dependencies
|
- name: Install node dependencies
|
||||||
run: make V=1 node.env
|
run: make V=1 node.env
|
||||||
- name: Build themes
|
- name: Build themes
|
||||||
run: make V=1 themes.all
|
run: make V=1 themes.all
|
||||||
|
|
||||||
documentation:
|
documentation:
|
||||||
name: Documentation
|
name: Documentation
|
||||||
|
|
@ -79,40 +79,40 @@ jobs:
|
||||||
permissions:
|
permissions:
|
||||||
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo
|
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: '0'
|
fetch-depth: '0'
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- name: Install Ubuntu packages
|
- name: Install Ubuntu packages
|
||||||
run: sudo ./utils/searxng.sh install buildhost
|
run: sudo ./utils/searxng.sh install buildhost
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||||
- name: Build documentation
|
- name: Build documentation
|
||||||
run: |
|
run: |
|
||||||
make V=1 docs.clean docs.html
|
make V=1 docs.clean docs.html
|
||||||
- name: Deploy
|
- name: Deploy
|
||||||
if: github.ref == 'refs/heads/master'
|
if: github.ref == 'refs/heads/master'
|
||||||
uses: JamesIves/github-pages-deploy-action@3.7.1
|
uses: JamesIves/github-pages-deploy-action@3.7.1
|
||||||
with:
|
with:
|
||||||
GITHUB_TOKEN: ${{ github.token }}
|
GITHUB_TOKEN: ${{ github.token }}
|
||||||
BRANCH: gh-pages
|
BRANCH: gh-pages
|
||||||
FOLDER: dist/docs
|
FOLDER: dist/docs
|
||||||
CLEAN: true # Automatically remove deleted files from the deploy branch
|
CLEAN: true # Automatically remove deleted files from the deploy branch
|
||||||
SINGLE_COMMIT: True
|
SINGLE_COMMIT: true
|
||||||
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
|
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
|
||||||
|
|
||||||
babel:
|
babel:
|
||||||
name: Update translations branch
|
name: Update translations branch
|
||||||
|
|
@ -125,37 +125,37 @@ jobs:
|
||||||
permissions:
|
permissions:
|
||||||
contents: write # for make V=1 weblate.push.translations
|
contents: write # for make V=1 weblate.push.translations
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: '0'
|
fetch-depth: '0'
|
||||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||||
- name: weblate & git setup
|
- name: weblate & git setup
|
||||||
env:
|
env:
|
||||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p ~/.config
|
mkdir -p ~/.config
|
||||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||||
git config --global user.name "searxng-bot"
|
git config --global user.name "searxng-bot"
|
||||||
- name: Update transations
|
- name: Update transations
|
||||||
id: update
|
id: update
|
||||||
run: |
|
run: |
|
||||||
make V=1 weblate.push.translations
|
make V=1 weblate.push.translations
|
||||||
|
|
||||||
dockers:
|
dockers:
|
||||||
name: Docker
|
name: Docker
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Security checks"
|
name: "Security checks"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "42 05 * * *"
|
- cron: "42 05 * * *"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Update translations"
|
name: "Update translations"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "05 07 * * 5"
|
- cron: "05 07 * * 5"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
@ -10,50 +10,50 @@ jobs:
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: '0'
|
fetch-depth: '0'
|
||||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
python-version: '3.12'
|
||||||
architecture: 'x64'
|
architecture: 'x64'
|
||||||
- name: Cache Python dependencies
|
- name: Cache Python dependencies
|
||||||
id: cache-python
|
id: cache-python
|
||||||
uses: actions/cache@v3
|
uses: actions/cache@v3
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
./local
|
./local
|
||||||
./.nvm
|
./.nvm
|
||||||
./node_modules
|
./node_modules
|
||||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||||
- name: weblate & git setup
|
- name: weblate & git setup
|
||||||
env:
|
env:
|
||||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p ~/.config
|
mkdir -p ~/.config
|
||||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||||
git config --global user.name "searxng-bot"
|
git config --global user.name "searxng-bot"
|
||||||
- name: Merge and push transation updates
|
- name: Merge and push transation updates
|
||||||
run: |
|
run: |
|
||||||
make V=1 weblate.translations.commit
|
make V=1 weblate.translations.commit
|
||||||
- name: Create Pull Request
|
- name: Create Pull Request
|
||||||
id: cpr
|
id: cpr
|
||||||
uses: peter-evans/create-pull-request@v3
|
uses: peter-evans/create-pull-request@v3
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||||
commit-message: '[l10n] update translations from Weblate'
|
commit-message: '[l10n] update translations from Weblate'
|
||||||
committer: searxng-bot <searxng-bot@users.noreply.github.com>
|
committer: searxng-bot <searxng-bot@users.noreply.github.com>
|
||||||
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
||||||
signoff: false
|
signoff: false
|
||||||
branch: translations_update
|
branch: translations_update
|
||||||
delete-branch: true
|
delete-branch: true
|
||||||
draft: false
|
draft: false
|
||||||
title: '[l10n] update translations from Weblate'
|
title: '[l10n] update translations from Weblate'
|
||||||
body: |
|
body: |
|
||||||
update translations from Weblate
|
update translations from Weblate
|
||||||
labels: |
|
labels: |
|
||||||
translation
|
translation
|
||||||
|
|
|
||||||
2
manage
2
manage
|
|
@ -57,7 +57,7 @@ while IFS= read -r line; do
|
||||||
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
|
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
|
||||||
YAMLLINT_FILES+=("$line")
|
YAMLLINT_FILES+=("$line")
|
||||||
fi
|
fi
|
||||||
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
|
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
|
||||||
|
|
||||||
RST_FILES=(
|
RST_FILES=(
|
||||||
'README.rst'
|
'README.rst'
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ cov-core==1.15.0
|
||||||
black==24.3.0
|
black==24.3.0
|
||||||
pylint==3.3.1
|
pylint==3.3.1
|
||||||
splinter==0.21.0
|
splinter==0.21.0
|
||||||
selenium==4.25.0
|
selenium==4.26.1
|
||||||
Pallets-Sphinx-Themes==2.3.0
|
Pallets-Sphinx-Themes==2.3.0
|
||||||
Sphinx==7.4.7
|
Sphinx==7.4.7
|
||||||
sphinx-issues==5.0.0
|
sphinx-issues==5.0.0
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,11 @@
|
||||||
# pylint: disable=missing-module-docstring, invalid-name
|
# pylint: disable=missing-module-docstring, invalid-name
|
||||||
|
|
||||||
import gc
|
import gc
|
||||||
|
import time
|
||||||
import typing
|
import typing
|
||||||
import types
|
import types
|
||||||
import functools
|
import functools
|
||||||
import itertools
|
import itertools
|
||||||
from time import time
|
|
||||||
from timeit import default_timer
|
from timeit import default_timer
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
|
@ -66,15 +66,15 @@ def _download_and_check_if_image(image_url: str) -> bool:
|
||||||
This function should not be called directly: use _is_url_image
|
This function should not be called directly: use _is_url_image
|
||||||
otherwise the cache of functools.lru_cache contains data: URL which might be huge.
|
otherwise the cache of functools.lru_cache contains data: URL which might be huge.
|
||||||
"""
|
"""
|
||||||
retry = 2
|
retry = 3
|
||||||
|
|
||||||
while retry > 0:
|
while retry > 0:
|
||||||
a = time()
|
a = default_timer()
|
||||||
try:
|
try:
|
||||||
# use "image_proxy" (avoid HTTP/2)
|
# use "image_proxy" (avoid HTTP/2)
|
||||||
|
network.set_timeout_for_thread(2)
|
||||||
network.set_context_network_name('image_proxy')
|
network.set_context_network_name('image_proxy')
|
||||||
r, stream = network.stream(
|
r = network.get(
|
||||||
'GET',
|
|
||||||
image_url,
|
image_url,
|
||||||
timeout=10.0,
|
timeout=10.0,
|
||||||
allow_redirects=True,
|
allow_redirects=True,
|
||||||
|
|
@ -90,19 +90,20 @@ def _download_and_check_if_image(image_url: str) -> bool:
|
||||||
'Cache-Control': 'max-age=0',
|
'Cache-Control': 'max-age=0',
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
r.close()
|
|
||||||
if r.status_code == 200:
|
if r.status_code == 200:
|
||||||
is_image = r.headers.get('content-type', '').startswith('image/')
|
is_image = r.headers.get('content-type', '').startswith('image/')
|
||||||
else:
|
else:
|
||||||
is_image = False
|
is_image = False
|
||||||
del r
|
del r
|
||||||
del stream
|
|
||||||
return is_image
|
return is_image
|
||||||
except httpx.TimeoutException:
|
except httpx.TimeoutException:
|
||||||
logger.error('Timeout for %s: %i', image_url, int(time() - a))
|
logger.error('Timeout for %s: %i', image_url, int(default_timer() - a))
|
||||||
|
time.sleep(1)
|
||||||
retry -= 1
|
retry -= 1
|
||||||
except httpx.HTTPError:
|
except httpx.HTTPStatusError as e:
|
||||||
logger.exception('Exception for %s', image_url)
|
logger.error('Exception for %s: HTTP status=%i', image_url, e.response.status_code)
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
logger.error('Exception for %s: %s, %s', image_url, e.__class__.__name__, ",".join(e.args))
|
||||||
return False
|
return False
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue