Compare commits

...

5 Commits

Author SHA1 Message Date
Alexandre Flament aa9c5b4d9b
Merge f49d1a9b90 into cd384a8a60 2024-11-06 10:03:21 +01:00
dependabot[bot] cd384a8a60 [upd] pypi: Bump selenium from 4.25.0 to 4.26.1
Bumps [selenium](https://github.com/SeleniumHQ/Selenium) from 4.25.0 to 4.26.1.
- [Release notes](https://github.com/SeleniumHQ/Selenium/releases)
- [Commits](https://github.com/SeleniumHQ/Selenium/commits)

---
updated-dependencies:
- dependency-name: selenium
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-11-06 10:01:13 +01:00
Markus Heiser c4055e449f [fix] issues reported by `make test.yamllint`
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2024-11-06 08:16:21 +01:00
Markus Heiser 2fdbf2622b [mod] lint github YAML config files
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
2024-11-06 08:16:21 +01:00
Alexandre FLAMENT f49d1a9b90 Checker: change image check
In the master branch, the checker starts to stream the response
and cut the connection. However it creates a lot of read error,
which are false negative. I don't know how to fix the issue.

This commit change the checker to download the whole image.
The error reporting is also changed to report only one line,
instead of the whole stacktrace.

Also, if a timeout occurs, the checker waits for one second
before retry.

Do note that I have not test the checker running in background.
This feature seems forgotten and lack of interrest despite the
initial move few years ago.
2024-03-09 09:57:19 +00:00
8 changed files with 180 additions and 179 deletions

View File

@ -1,5 +1,5 @@
name: "Checker" name: "Checker"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "0 4 * * 5" - cron: "0 4 * * 5"
workflow_dispatch: workflow_dispatch:

View File

@ -1,5 +1,5 @@
name: "Update searx.data" name: "Update searx.data"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "59 23 28 * *" - cron: "59 23 28 * *"
workflow_dispatch: workflow_dispatch:

View File

@ -1,6 +1,6 @@
name: Integration name: Integration
on: on: # yamllint disable-line rule:truthy
push: push:
branches: ["master"] branches: ["master"]
pull_request: pull_request:
@ -16,7 +16,7 @@ jobs:
strategy: strategy:
matrix: matrix:
os: [ubuntu-20.04] os: [ubuntu-20.04]
python-version: ["3.9", "3.10", "3.11", "3.12",] python-version: ["3.9", "3.10", "3.11", "3.12"]
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
@ -111,7 +111,7 @@ jobs:
BRANCH: gh-pages BRANCH: gh-pages
FOLDER: dist/docs FOLDER: dist/docs
CLEAN: true # Automatically remove deleted files from the deploy branch CLEAN: true # Automatically remove deleted files from the deploy branch
SINGLE_COMMIT: True SINGLE_COMMIT: true
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}' COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
babel: babel:

View File

@ -1,5 +1,5 @@
name: "Security checks" name: "Security checks"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "42 05 * * *" - cron: "42 05 * * *"
workflow_dispatch: workflow_dispatch:

View File

@ -1,5 +1,5 @@
name: "Update translations" name: "Update translations"
on: on: # yamllint disable-line rule:truthy
schedule: schedule:
- cron: "05 07 * * 5" - cron: "05 07 * * 5"
workflow_dispatch: workflow_dispatch:

2
manage
View File

@ -57,7 +57,7 @@ while IFS= read -r line; do
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
YAMLLINT_FILES+=("$line") YAMLLINT_FILES+=("$line")
fi fi
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')" done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
RST_FILES=( RST_FILES=(
'README.rst' 'README.rst'

View File

@ -4,7 +4,7 @@ cov-core==1.15.0
black==24.3.0 black==24.3.0
pylint==3.3.1 pylint==3.3.1
splinter==0.21.0 splinter==0.21.0
selenium==4.25.0 selenium==4.26.1
Pallets-Sphinx-Themes==2.3.0 Pallets-Sphinx-Themes==2.3.0
Sphinx==7.4.7 Sphinx==7.4.7
sphinx-issues==5.0.0 sphinx-issues==5.0.0

View File

@ -2,11 +2,11 @@
# pylint: disable=missing-module-docstring, invalid-name # pylint: disable=missing-module-docstring, invalid-name
import gc import gc
import time
import typing import typing
import types import types
import functools import functools
import itertools import itertools
from time import time
from timeit import default_timer from timeit import default_timer
from urllib.parse import urlparse from urllib.parse import urlparse
@ -66,15 +66,15 @@ def _download_and_check_if_image(image_url: str) -> bool:
This function should not be called directly: use _is_url_image This function should not be called directly: use _is_url_image
otherwise the cache of functools.lru_cache contains data: URL which might be huge. otherwise the cache of functools.lru_cache contains data: URL which might be huge.
""" """
retry = 2 retry = 3
while retry > 0: while retry > 0:
a = time() a = default_timer()
try: try:
# use "image_proxy" (avoid HTTP/2) # use "image_proxy" (avoid HTTP/2)
network.set_timeout_for_thread(2)
network.set_context_network_name('image_proxy') network.set_context_network_name('image_proxy')
r, stream = network.stream( r = network.get(
'GET',
image_url, image_url,
timeout=10.0, timeout=10.0,
allow_redirects=True, allow_redirects=True,
@ -90,19 +90,20 @@ def _download_and_check_if_image(image_url: str) -> bool:
'Cache-Control': 'max-age=0', 'Cache-Control': 'max-age=0',
}, },
) )
r.close()
if r.status_code == 200: if r.status_code == 200:
is_image = r.headers.get('content-type', '').startswith('image/') is_image = r.headers.get('content-type', '').startswith('image/')
else: else:
is_image = False is_image = False
del r del r
del stream
return is_image return is_image
except httpx.TimeoutException: except httpx.TimeoutException:
logger.error('Timeout for %s: %i', image_url, int(time() - a)) logger.error('Timeout for %s: %i', image_url, int(default_timer() - a))
time.sleep(1)
retry -= 1 retry -= 1
except httpx.HTTPError: except httpx.HTTPStatusError as e:
logger.exception('Exception for %s', image_url) logger.error('Exception for %s: HTTP status=%i', image_url, e.response.status_code)
except httpx.HTTPError as e:
logger.error('Exception for %s: %s, %s', image_url, e.__class__.__name__, ",".join(e.args))
return False return False
return False return False