mirror of https://github.com/searxng/searxng.git
Compare commits
5 Commits
9b1cbb3218
...
aa9c5b4d9b
Author | SHA1 | Date |
---|---|---|
Alexandre Flament | aa9c5b4d9b | |
dependabot[bot] | cd384a8a60 | |
Markus Heiser | c4055e449f | |
Markus Heiser | 2fdbf2622b | |
Alexandre FLAMENT | f49d1a9b90 |
|
@ -1,5 +1,5 @@
|
||||||
name: "Checker"
|
name: "Checker"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "0 4 * * 5"
|
- cron: "0 4 * * 5"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Update searx.data"
|
name: "Update searx.data"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "59 23 28 * *"
|
- cron: "59 23 28 * *"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
name: Integration
|
name: Integration
|
||||||
|
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
push:
|
push:
|
||||||
branches: ["master"]
|
branches: ["master"]
|
||||||
pull_request:
|
pull_request:
|
||||||
|
@ -16,7 +16,7 @@ jobs:
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
os: [ubuntu-20.04]
|
os: [ubuntu-20.04]
|
||||||
python-version: ["3.9", "3.10", "3.11", "3.12",]
|
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
@ -111,7 +111,7 @@ jobs:
|
||||||
BRANCH: gh-pages
|
BRANCH: gh-pages
|
||||||
FOLDER: dist/docs
|
FOLDER: dist/docs
|
||||||
CLEAN: true # Automatically remove deleted files from the deploy branch
|
CLEAN: true # Automatically remove deleted files from the deploy branch
|
||||||
SINGLE_COMMIT: True
|
SINGLE_COMMIT: true
|
||||||
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
|
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
|
||||||
|
|
||||||
babel:
|
babel:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Security checks"
|
name: "Security checks"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "42 05 * * *"
|
- cron: "42 05 * * *"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
name: "Update translations"
|
name: "Update translations"
|
||||||
on:
|
on: # yamllint disable-line rule:truthy
|
||||||
schedule:
|
schedule:
|
||||||
- cron: "05 07 * * 5"
|
- cron: "05 07 * * 5"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
2
manage
2
manage
|
@ -57,7 +57,7 @@ while IFS= read -r line; do
|
||||||
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
|
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
|
||||||
YAMLLINT_FILES+=("$line")
|
YAMLLINT_FILES+=("$line")
|
||||||
fi
|
fi
|
||||||
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
|
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
|
||||||
|
|
||||||
RST_FILES=(
|
RST_FILES=(
|
||||||
'README.rst'
|
'README.rst'
|
||||||
|
|
|
@ -4,7 +4,7 @@ cov-core==1.15.0
|
||||||
black==24.3.0
|
black==24.3.0
|
||||||
pylint==3.3.1
|
pylint==3.3.1
|
||||||
splinter==0.21.0
|
splinter==0.21.0
|
||||||
selenium==4.25.0
|
selenium==4.26.1
|
||||||
Pallets-Sphinx-Themes==2.3.0
|
Pallets-Sphinx-Themes==2.3.0
|
||||||
Sphinx==7.4.7
|
Sphinx==7.4.7
|
||||||
sphinx-issues==5.0.0
|
sphinx-issues==5.0.0
|
||||||
|
|
|
@ -2,11 +2,11 @@
|
||||||
# pylint: disable=missing-module-docstring, invalid-name
|
# pylint: disable=missing-module-docstring, invalid-name
|
||||||
|
|
||||||
import gc
|
import gc
|
||||||
|
import time
|
||||||
import typing
|
import typing
|
||||||
import types
|
import types
|
||||||
import functools
|
import functools
|
||||||
import itertools
|
import itertools
|
||||||
from time import time
|
|
||||||
from timeit import default_timer
|
from timeit import default_timer
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
@ -66,15 +66,15 @@ def _download_and_check_if_image(image_url: str) -> bool:
|
||||||
This function should not be called directly: use _is_url_image
|
This function should not be called directly: use _is_url_image
|
||||||
otherwise the cache of functools.lru_cache contains data: URL which might be huge.
|
otherwise the cache of functools.lru_cache contains data: URL which might be huge.
|
||||||
"""
|
"""
|
||||||
retry = 2
|
retry = 3
|
||||||
|
|
||||||
while retry > 0:
|
while retry > 0:
|
||||||
a = time()
|
a = default_timer()
|
||||||
try:
|
try:
|
||||||
# use "image_proxy" (avoid HTTP/2)
|
# use "image_proxy" (avoid HTTP/2)
|
||||||
|
network.set_timeout_for_thread(2)
|
||||||
network.set_context_network_name('image_proxy')
|
network.set_context_network_name('image_proxy')
|
||||||
r, stream = network.stream(
|
r = network.get(
|
||||||
'GET',
|
|
||||||
image_url,
|
image_url,
|
||||||
timeout=10.0,
|
timeout=10.0,
|
||||||
allow_redirects=True,
|
allow_redirects=True,
|
||||||
|
@ -90,19 +90,20 @@ def _download_and_check_if_image(image_url: str) -> bool:
|
||||||
'Cache-Control': 'max-age=0',
|
'Cache-Control': 'max-age=0',
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
r.close()
|
|
||||||
if r.status_code == 200:
|
if r.status_code == 200:
|
||||||
is_image = r.headers.get('content-type', '').startswith('image/')
|
is_image = r.headers.get('content-type', '').startswith('image/')
|
||||||
else:
|
else:
|
||||||
is_image = False
|
is_image = False
|
||||||
del r
|
del r
|
||||||
del stream
|
|
||||||
return is_image
|
return is_image
|
||||||
except httpx.TimeoutException:
|
except httpx.TimeoutException:
|
||||||
logger.error('Timeout for %s: %i', image_url, int(time() - a))
|
logger.error('Timeout for %s: %i', image_url, int(default_timer() - a))
|
||||||
|
time.sleep(1)
|
||||||
retry -= 1
|
retry -= 1
|
||||||
except httpx.HTTPError:
|
except httpx.HTTPStatusError as e:
|
||||||
logger.exception('Exception for %s', image_url)
|
logger.error('Exception for %s: HTTP status=%i', image_url, e.response.status_code)
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
logger.error('Exception for %s: %s, %s', image_url, e.__class__.__name__, ",".join(e.args))
|
||||||
return False
|
return False
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue