Merge 1c9b28968d into 56e3d72a76

The test.coverage cause a lot of failed CI jobs for reasons that cannot be
explained.  As we do not monitor the coverage anyway, it is superfluous to run
this job, especially as it only has a disruptive effect on the CI.

BTW and the CI action upload-artifact@v3 is deprecated [1]

[1] https://github.com/actions/upload-artifact?tab=readme-ov-file#actionsupload-artifact

Related: https://github.com/searxng/searxng/issues/3983
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

.github/workflows/integration.yml

@@ -45,14 +45,6 @@ jobs:
         make V=1 gecko.driver
     - name: Run tests
       run: make V=1 ci.test
-    - name: Test coverage
-      run: make V=1 test.coverage
-    - name: Store coverage result
-      uses: actions/upload-artifact@v3
-      with:
-        name: coverage-${{ matrix.python-version }}
-        path: coverage/
-        retention-days: 60
 
   themes:
     name: Themes

docs/src/searx.botdetection.rst

@@ -53,6 +53,9 @@ Probe HTTP headers
 .. automodule:: searx.botdetection.http_user_agent
   :members:
 
+.. automodule:: searx.botdetection.sec_fetch
+  :members:
+
 .. _botdetection config:
 
 Config

searx/botdetection/_helpers.py

@@ -31,6 +31,9 @@ def dump_request(request: flask.Request):
         + " || Content-Length: %s" % request.headers.get('Content-Length')
         + " || Connection: %s" % request.headers.get('Connection')
         + " || User-Agent: %s" % request.headers.get('User-Agent')
+        + " || Sec-Fetch-Site: %s" % request.headers.get('Sec-Fetch-Site')
+        + " || Sec-Fetch-Mode: %s" % request.headers.get('Sec-Fetch-Mode')
+        + " || Sec-Fetch-Dest: %s" % request.headers.get('Sec-Fetch-Dest')
     )
 
 

searx/botdetection/http_sec_fetch.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+Method ``http_sec_fetch``
+-------------------------
+
+The ``http_sec_fetch`` method protect resources from web attacks with `Fetch
+Metadata`_.  A request is filtered out in case of:
+
+- http header Sec-Fetch-Mode_ is invalid
+- http header Sec-Fetch-Dest_ is invalid
+
+.. _Fetch Metadata:
+   https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header
+
+.. Sec-Fetch-Dest:
+   https://developer.mozilla.org/en-US/docs/Web/API/Request/destination
+
+.. Sec-Fetch-Mode:
+   https://developer.mozilla.org/en-US/docs/Web/API/Request/mode
+
+
+"""
+# pylint: disable=unused-argument
+
+from __future__ import annotations
+from ipaddress import (
+    IPv4Network,
+    IPv6Network,
+)
+
+import flask
+import werkzeug
+
+from . import config
+from ._helpers import logger
+
+
+def filter_request(
+    network: IPv4Network | IPv6Network,
+    request: flask.Request,
+    cfg: config.Config,
+) -> werkzeug.Response | None:
+
+    val = request.headers.get("Sec-Fetch-Mode", "")
+    if val != "navigate":
+        logger.debug("invalid Sec-Fetch-Mode '%s'", val)
+        return flask.redirect(flask.url_for('index'), code=302)
+
+    val = request.headers.get("Sec-Fetch-Site", "")
+    if val not in ('same-origin', 'same-site', 'none'):
+        logger.debug("invalid Sec-Fetch-Site '%s'", val)
+        flask.redirect(flask.url_for('index'), code=302)
+
+    val = request.headers.get("Sec-Fetch-Dest", "")
+    if val != "document":
+        logger.debug("invalid Sec-Fetch-Dest '%s'", val)
+        flask.redirect(flask.url_for('index'), code=302)
+
+    return None

searx/limiter.py

@@ -111,6 +111,7 @@ from searx.botdetection import (
     http_accept_encoding,
     http_accept_language,
     http_user_agent,
+    http_sec_fetch,
     ip_limit,
     ip_lists,
     get_network,
@@ -178,16 +179,17 @@ def filter_request(request: flask.Request) -> werkzeug.Response | None:
         logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg)
         return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429))
 
-    # methods applied on /
+    # methods applied on all requests
 
     for func in [
         http_user_agent,
     ]:
         val = func.filter_request(network, request, cfg)
         if val is not None:
+            logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(flask.request))
             return val
 
-    # methods applied on /search
+    # methods applied on /search requests
 
     if request.path == '/search':
 
@@ -196,12 +198,15 @@ def filter_request(request: flask.Request) -> werkzeug.Response | None:
             http_accept_encoding,
             http_accept_language,
             http_user_agent,
+            http_sec_fetch,
             ip_limit,
         ]:
             val = func.filter_request(network, request, cfg)
             if val is not None:
+                logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(flask.request))
                 return val
-    logger.debug(f"OK {network}: %s", dump_request(flask.request))
+        logger.debug(f"OK: {network}: %s", dump_request(flask.request))
+
     return None
 
 

searx/translations/es/LC_MESSAGES/messages.po

@@ -39,7 +39,7 @@ msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-06 14:31+0000\n"
+"PO-Revision-Date: 2024-10-26 21:13+0000\n"
 "Last-Translator: Atul_Eterno <Atul_Eterno@users.noreply.translate.codeberg."
 "org>\n"
 "Language-Team: Spanish <https://translate.codeberg.org/projects/searxng/"
@@ -49,7 +49,7 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
-"X-Generator: Weblate 5.7.2\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -155,7 +155,7 @@ msgstr "preguntas y respuestas"
 #. CATEGORY_GROUPS['REPOS']
 #: searx/searxng.msg
 msgid "repos"
-msgstr "repos"
+msgstr "repositorios"
 
 #. CATEGORY_GROUPS['SOFTWARE_WIKIS']
 #: searx/searxng.msg

searx/translations/nl/LC_MESSAGES/messages.po

@@ -22,13 +22,14 @@
 # MVDW-Java <MVDW-Java@users.noreply.translate.codeberg.org>, 2024.
 # notlmutsaers <notlmutsaers@users.noreply.translate.codeberg.org>, 2024.
 # return42 <return42@users.noreply.translate.codeberg.org>, 2024.
+# ljansen <ljansen@users.noreply.translate.codeberg.org>, 2024.
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-15 12:18+0000\n"
-"Last-Translator: return42 <return42@users.noreply.translate.codeberg.org>\n"
+"PO-Revision-Date: 2024-10-28 21:07+0000\n"
+"Last-Translator: ljansen <ljansen@users.noreply.translate.codeberg.org>\n"
 "Language-Team: Dutch <https://translate.codeberg.org/projects/searxng/"
 "searxng/nl/>\n"
 "Language: nl\n"
@@ -36,7 +37,7 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
-"X-Generator: Weblate 5.7.2\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -464,7 +465,7 @@ msgstr "Bereken {functions} van de opties"
 
 #: searx/engines/mozhi.py:57
 msgid "Synonyms"
-msgstr ""
+msgstr "Synoniemen"
 
 #: searx/engines/openstreetmap.py:159
 msgid "Get directions"
@@ -1234,12 +1235,13 @@ msgid "Max time"
 msgstr "Max. duur"
 
 #: searx/templates/simple/preferences/favicon.html:2
+#, fuzzy
 msgid "Favicon Resolver"
-msgstr ""
+msgstr "favicon-resolver"
 
 #: searx/templates/simple/preferences/favicon.html:15
 msgid "Display favicons near search results"
-msgstr ""
+msgstr "Vertoon zoekresultaten naast favicons"
 
 #: searx/templates/simple/preferences/footer.html:2
 msgid ""

searx/translations/pl/LC_MESSAGES/messages.po

@@ -23,8 +23,8 @@ msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-15 12:18+0000\n"
-"Last-Translator: return42 <return42@users.noreply.translate.codeberg.org>\n"
+"PO-Revision-Date: 2024-10-28 21:07+0000\n"
+"Last-Translator: Eryk Michalak <gnu.ewm@protonmail.com>\n"
 "Language-Team: Polish <https://translate.codeberg.org/projects/searxng/"
 "searxng/pl/>\n"
 "Language: pl\n"
@@ -34,7 +34,7 @@ msgstr ""
 "Plural-Forms: nplurals=4; plural=(n==1 ? 0 : (n%10>=2 && n%10<=4) && ("
 "n%100<12 || n%100>14) ? 1 : n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && "
 "n%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3);\n"
-"X-Generator: Weblate 5.7.2\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -1230,7 +1230,7 @@ msgstr "Maksymalny czas"
 
 #: searx/templates/simple/preferences/favicon.html:2
 msgid "Favicon Resolver"
-msgstr ""
+msgstr "Pobieranie favikony"
 
 #: searx/templates/simple/preferences/favicon.html:15
 msgid "Display favicons near search results"

searx/translations/pt/LC_MESSAGES/messages.po

@@ -19,13 +19,14 @@
 # gvlx <gvlx@users.noreply.translate.codeberg.org>, 2024.
 # ds451 <ds451@users.noreply.translate.codeberg.org>, 2024.
 # Pedro_Tresp <Pedro_Tresp@users.noreply.translate.codeberg.org>, 2024.
+# saltsnorter <saltsnorter@users.noreply.translate.codeberg.org>, 2024.
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-13 23:26+0000\n"
-"Last-Translator: Pedro_Tresp <Pedro_Tresp@users.noreply.translate.codeberg."
+"PO-Revision-Date: 2024-10-29 05:54+0000\n"
+"Last-Translator: saltsnorter <saltsnorter@users.noreply.translate.codeberg."
 "org>\n"
 "Language-Team: Portuguese <https://translate.codeberg.org/projects/searxng/"
 "searxng/pt/>\n"
@@ -34,7 +35,7 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
-"X-Generator: Weblate 5.7.2\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -175,7 +176,7 @@ msgstr "escuro"
 #. STYLE_NAMES['BLACK']
 #: searx/searxng.msg
 msgid "black"
-msgstr ""
+msgstr "preto"
 
 #. BRAND_CUSTOM_LINKS['UPTIME']
 #: searx/searxng.msg
@@ -1228,11 +1229,11 @@ msgstr "Tempo máximo"
 
 #: searx/templates/simple/preferences/favicon.html:2
 msgid "Favicon Resolver"
-msgstr ""
+msgstr "Solucionador do Favicon"
 
 #: searx/templates/simple/preferences/favicon.html:15
 msgid "Display favicons near search results"
-msgstr ""
+msgstr "Monstra os favicons nos proximos os resultados"
 
 #: searx/templates/simple/preferences/footer.html:2
 msgid ""

searx/translations/pt_BR/LC_MESSAGES/messages.po

@@ -30,13 +30,14 @@
 # Pyrbor <Pyrbor@users.noreply.translate.codeberg.org>, 2024.
 # rodgui <rodgui@users.noreply.translate.codeberg.org>, 2024.
 # rafablog77 <rafablog77@users.noreply.translate.codeberg.org>, 2024.
+# Juno Takano <jutty@users.noreply.translate.codeberg.org>, 2024.
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-10-15 12:18+0000\n"
-"Last-Translator: return42 <return42@users.noreply.translate.codeberg.org>\n"
+"PO-Revision-Date: 2024-10-31 12:16+0000\n"
+"Last-Translator: Juno Takano <jutty@users.noreply.translate.codeberg.org>\n"
 "Language-Team: Portuguese (Brazil) <https://translate.codeberg.org/projects/"
 "searxng/searxng/pt_BR/>\n"
 "Language: pt_BR\n"
@@ -44,7 +45,7 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n > 1;\n"
-"X-Generator: Weblate 5.7.2\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -185,7 +186,7 @@ msgstr "escuro"
 #. STYLE_NAMES['BLACK']
 #: searx/searxng.msg
 msgid "black"
-msgstr ""
+msgstr "preto"
 
 #. BRAND_CUSTOM_LINKS['UPTIME']
 #: searx/searxng.msg
@@ -472,7 +473,7 @@ msgstr "Computar {functions} dos argumentos"
 
 #: searx/engines/mozhi.py:57
 msgid "Synonyms"
-msgstr ""
+msgstr "Sinônimos"
 
 #: searx/engines/openstreetmap.py:159
 msgid "Get directions"
@@ -1243,7 +1244,7 @@ msgstr "Tempo máximo"
 
 #: searx/templates/simple/preferences/favicon.html:2
 msgid "Favicon Resolver"
-msgstr ""
+msgstr "Resolvedor de Favicons"
 
 #: searx/templates/simple/preferences/favicon.html:15
 msgid "Display favicons near search results"

searx/translations/vi/LC_MESSAGES/messages.po

@@ -12,19 +12,19 @@
 # tvminh19 <tvminh19@users.noreply.translate.codeberg.org>, 2024.
 msgid ""
 msgstr ""
-"Project-Id-Version:  searx\n"
+"Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-08-07 01:02+0000\n"
-"Last-Translator: tvminh19 <tvminh19@users.noreply.translate.codeberg.org>"
-"\n"
+"PO-Revision-Date: 2024-10-26 21:13+0000\n"
+"Last-Translator: return42 <return42@users.noreply.translate.codeberg.org>\n"
+"Language-Team: Vietnamese <https://translate.codeberg.org/projects/searxng/"
+"searxng/vi/>\n"
 "Language: vi\n"
-"Language-Team: Vietnamese "
-"<https://translate.codeberg.org/projects/searxng/searxng/vi/>\n"
-"Plural-Forms: nplurals=1; plural=0;\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=1; plural=0;\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -186,7 +186,7 @@ msgstr "Nhiệt độ trung bình."
 #. WEATHER_TERMS['CLOUD COVER']
 #: searx/engines/open_meteo.py:91 searx/searxng.msg
 msgid "Cloud cover"
-msgstr ""
+msgstr "Mây che phủ"
 
 #. WEATHER_TERMS['CONDITION']
 #: searx/engines/duckduckgo_weather.py:45 searx/engines/wttr.py:51
@@ -283,7 +283,7 @@ msgstr ""
 #: searx/engines/duckduckgo_weather.py:58 searx/engines/open_meteo.py:86
 #: searx/engines/wttr.py:62 searx/searxng.msg
 msgid "Wind"
-msgstr ""
+msgstr "Gió"
 
 #. SOCIAL_MEDIA_TERMS['SUBSCRIBERS']
 #: searx/engines/lemmy.py:85 searx/searxng.msg
@@ -1990,4 +1990,3 @@ msgstr "ẩn phim"
 
 #~ msgid "Engines cannot retrieve results"
 #~ msgstr "Các trình tìm kiếm không nhận được kết quả"
-

searx/translations/zh_Hant_TW/LC_MESSAGES/messages.po

@@ -28,18 +28,19 @@
 # hugoalh <hugoalh@users.noreply.translate.codeberg.org>, 2024.
 msgid ""
 msgstr ""
-"Project-Id-Version:  searx\n"
+"Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2024-10-05 06:24+0000\n"
-"PO-Revision-Date: 2024-08-12 04:00+0000\n"
-"Last-Translator: hugoalh <hugoalh@users.noreply.translate.codeberg.org>\n"
+"PO-Revision-Date: 2024-10-26 21:13+0000\n"
+"Last-Translator: return42 <return42@users.noreply.translate.codeberg.org>\n"
+"Language-Team: Chinese (Traditional Han script) <https://translate.codeberg."
+"org/projects/searxng/searxng/zh_Hant/>\n"
 "Language: zh_Hant_TW\n"
-"Language-Team: Chinese (Traditional) "
-"<https://translate.codeberg.org/projects/searxng/searxng/zh_Hant/>\n"
-"Plural-Forms: nplurals=1; plural=0;\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=1; plural=0;\n"
+"X-Generator: Weblate 5.8.1\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -180,7 +181,7 @@ msgstr "黑暗"
 #. STYLE_NAMES['BLACK']
 #: searx/searxng.msg
 msgid "black"
-msgstr ""
+msgstr "黑色"
 
 #. BRAND_CUSTOM_LINKS['UPTIME']
 #: searx/searxng.msg
@@ -467,7 +468,7 @@ msgstr "計算 {functions} 參數"
 
 #: searx/engines/mozhi.py:57
 msgid "Synonyms"
-msgstr ""
+msgstr "同義詞"
 
 #: searx/engines/openstreetmap.py:159
 msgid "Get directions"
@@ -942,7 +943,7 @@ msgstr "來自搜尋引擎的訊息"
 
 #: searx/templates/simple/elements/engines_msg.html:7
 msgid "seconds"
-msgstr ""
+msgstr "秒"
 
 #: searx/templates/simple/elements/search_url.html:3
 msgid "Search URL"
@@ -1206,11 +1207,11 @@ msgstr "最大時間"
 
 #: searx/templates/simple/preferences/favicon.html:2
 msgid "Favicon Resolver"
-msgstr ""
+msgstr "網站圖標搜索器"
 
 #: searx/templates/simple/preferences/favicon.html:15
 msgid "Display favicons near search results"
-msgstr ""
+msgstr "在搜尋結果旁顯示網站圖標"
 
 #: searx/templates/simple/preferences/footer.html:2
 msgid ""
@@ -1911,4 +1912,3 @@ msgstr "隱藏影片"
 
 #~ msgid "Engines cannot retrieve results"
 #~ msgstr "引擎無法擷取結果"
-