Merge 1c9b28968d into b183e620d8

The entire source code of the duckduckgo engine has been reengineered and
purified.

1. DDG used the URL https://html.duckduckgo.com/html for no-JS requests whose
   response is also easier to parse than the previous
   https://lite.duckduckgo.com/lite/ URL

2. the bot detection of DDG has so far caused problems and often led to a
   CAPTCHA, this can be circumvented using `'Sec-Fetch-Mode'] = “navigate”`

Closes: https://github.com/searxng/searxng/issues/3927
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

Dockerfile

@@ -43,16 +43,9 @@ RUN apk add --no-cache -t build-dependencies \
     tini \
     uwsgi \
     uwsgi-python3 \
-    brotli
+    brotli \
-
+ && pip3 install --break-system-packages --no-cache -r requirements.txt \
-# For 32bit arm architecture install pydantic from the alpine repos instead of requirements.txt
+ && apk del build-dependencies \
-ARG TARGETARCH
-RUN if [ "$TARGETARCH" = "arm" ]; then \
-        apk add --no-cache py3-pydantic && pip install --no-cache --break-system-packages -r <(grep -v '^pydantic' requirements.txt); \
-    else \
-        pip install --no-cache --break-system-packages -r requirements.txt; \
-    fi
- RUN apk del build-dependencies \
  && rm -rf /root/.cache
 
 COPY --chown=searxng:searxng dockerfiles ./dockerfiles

docs/src/searx.botdetection.rst

@@ -53,6 +53,9 @@ Probe HTTP headers
 .. automodule:: searx.botdetection.http_user_agent
   :members:
 
+.. automodule:: searx.botdetection.sec_fetch
+  :members:
+
 .. _botdetection config:
 
 Config

requirements.txt

@@ -16,6 +16,6 @@ redis==5.0.8
 markdown-it-py==3.0.0
 fasttext-predict==0.9.2.2
 tomli==2.0.2; python_version < '3.11'
-pydantic==2.9.2
+msgspec==0.18.6
 eval_type_backport; python_version < '3.9'
 typer-slim==0.12.5

searx/botdetection/_helpers.py

@@ -31,6 +31,9 @@ def dump_request(request: flask.Request):
         + " || Content-Length: %s" % request.headers.get('Content-Length')
         + " || Connection: %s" % request.headers.get('Connection')
         + " || User-Agent: %s" % request.headers.get('User-Agent')
+        + " || Sec-Fetch-Site: %s" % request.headers.get('Sec-Fetch-Site')
+        + " || Sec-Fetch-Mode: %s" % request.headers.get('Sec-Fetch-Mode')
+        + " || Sec-Fetch-Dest: %s" % request.headers.get('Sec-Fetch-Dest')
     )
 
 

searx/botdetection/http_sec_fetch.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+Method ``http_sec_fetch``
+-------------------------
+
+The ``http_sec_fetch`` method protect resources from web attacks with `Fetch
+Metadata`_.  A request is filtered out in case of:
+
+- http header Sec-Fetch-Mode_ is invalid
+- http header Sec-Fetch-Dest_ is invalid
+
+.. _Fetch Metadata:
+   https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header
+
+.. Sec-Fetch-Dest:
+   https://developer.mozilla.org/en-US/docs/Web/API/Request/destination
+
+.. Sec-Fetch-Mode:
+   https://developer.mozilla.org/en-US/docs/Web/API/Request/mode
+
+
+"""
+# pylint: disable=unused-argument
+
+from __future__ import annotations
+from ipaddress import (
+    IPv4Network,
+    IPv6Network,
+)
+
+import flask
+import werkzeug
+
+from . import config
+from ._helpers import logger
+
+
+def filter_request(
+    network: IPv4Network | IPv6Network,
+    request: flask.Request,
+    cfg: config.Config,
+) -> werkzeug.Response | None:
+
+    val = request.headers.get("Sec-Fetch-Mode", "")
+    if val != "navigate":
+        logger.debug("invalid Sec-Fetch-Mode '%s'", val)
+        return flask.redirect(flask.url_for('index'), code=302)
+
+    val = request.headers.get("Sec-Fetch-Site", "")
+    if val not in ('same-origin', 'same-site', 'none'):
+        logger.debug("invalid Sec-Fetch-Site '%s'", val)
+        flask.redirect(flask.url_for('index'), code=302)
+
+    val = request.headers.get("Sec-Fetch-Dest", "")
+    if val != "document":
+        logger.debug("invalid Sec-Fetch-Dest '%s'", val)
+        flask.redirect(flask.url_for('index'), code=302)
+
+    return None

searx/data/currencies.json

@@ -82,7 +82,7 @@
             "af": "Albanese lek",
             "ar": "ليك ألباني",
             "bg": "Албански лек",
-            "ca": "lek",
+            "ca": "Lek (moneda)",
             "cs": "Albánský lek",
             "cy": "Lek",
             "da": "Lek",
@@ -383,6 +383,7 @@
             "nl": "Azerbeidzjaanse manat",
             "oc": "Manat",
             "pa": "ਅਜ਼ਰਬਾਈਜਾਨੀ ਮਨਾਤ",
+            "pap": "Manat Azerbaijano",
             "pl": "Manat azerski",
             "pt": "Manat azeri",
             "ro": "Manat azer",
@@ -606,6 +607,7 @@
             "pt": "Franco do Burúndi",
             "ro": "franc burundez",
             "ru": "бурундийский франк",
+            "sk": "Burundský frank",
             "sl": "burundijski frank",
             "sr": "бурундски франак",
             "sv": "Burundisk franc",
@@ -1327,6 +1329,7 @@
             "pl": "escudo Zielonego Przylądka",
             "pt": "escudo cabo-verdiano",
             "ru": "Эскудо Кабо-Верде",
+            "sk": "Kapverdské escudo",
             "sl": "zelenortski eskudo",
             "sr": "зеленортски ескудо",
             "sv": "Kapverdisk escudo",
@@ -1405,6 +1408,7 @@
             "pl": "frank Dżibuti",
             "pt": "franco do Jibuti",
             "ru": "Франк Джибути",
+            "sk": "Džibutský frank",
             "sr": "џибутски франак",
             "sv": "Djiboutisk franc",
             "tr": "Cibuti frangı",
@@ -1518,6 +1522,7 @@
             "pt": "dinar argelino",
             "ro": "Dinar algerian",
             "ru": "алжирский динар",
+            "sk": "Alžírský dinár",
             "sl": "alžirski dinar",
             "sr": "алжирски динар",
             "sv": "Algerisk dinar",
@@ -1969,6 +1974,7 @@
             "pl": "frank gwinejski",
             "pt": "Franco da Guiné",
             "ru": "Гвинейский франк",
+            "sk": "Guinejský frank",
             "sl": "gvinejski frank",
             "sr": "гвинејски франак",
             "sv": "Guinesisk franc",
@@ -2689,6 +2695,7 @@
             "pt": "Franco comoriano",
             "ro": "Franc comorian",
             "ru": "Франк Комор",
+            "sk": "Komorský frank",
             "sr": "коморски франак",
             "sv": "Komoransk franc",
             "tr": "Komor frangı",
@@ -2986,6 +2993,7 @@
             "pt": "rúpia do Sri Lanka",
             "ru": "ланкийская рупия",
             "si": "ශ්රී ලංකා රුපියල",
+            "sk": "Srílanská rupia",
             "sl": "šrilanška rupija",
             "sr": "шриланчанска рупија",
             "sv": "Lankesisk rupie",
@@ -3059,7 +3067,7 @@
             "uk": "Лоті"
         },
         "LYD": {
-            "ar": "دينار ليبي",
+            "ar": "دينار ذهبي",
             "bg": "Либийски динар",
             "ca": "dinar libi",
             "cs": "Libyjský dinár",
@@ -3121,6 +3129,7 @@
             "pt": "Dirham marroquino",
             "ro": "Dirham marocan",
             "ru": "марокканский дирхам",
+            "sk": "Marocký dirham",
             "sl": "maroški dirham",
             "sr": "марокански дирхам",
             "sv": "Marockansk dirham",
@@ -3140,6 +3149,7 @@
             "et": "Moldova leu",
             "fi": "Moldovan leu",
             "fr": "leu moldave",
+            "gl": "leu moldovo",
             "he": "לאו מולדובני",
             "hr": "moldavski lej",
             "hu": "moldován lej",
@@ -3371,6 +3381,7 @@
             "pl": "Ugija",
             "pt": "Uguia",
             "ru": "Мавританская угия",
+            "sk": "Mauritánska ukíjá",
             "sr": "мауританска огија",
             "sv": "Mauretansk ouguiya",
             "tr": "Ugiya",
@@ -3816,6 +3827,7 @@
             "sl": "novozelandski dolar",
             "sr": "новозеландски долар",
             "sv": "Nyzeeländsk dollar",
+            "th": "ดอลลาร์นิวซีแลนด์",
             "tr": "Yeni Zelanda doları",
             "uk": "новозеландський долар",
             "vi": "Đô la New Zealand"
@@ -5386,12 +5398,14 @@
             "ja": "スム",
             "ko": "우즈베키스탄 숨",
             "lt": "Uzbekijos sumas",
+            "lv": "Uzbekistānas soms",
             "nl": "Oezbeekse sum",
             "pa": "ਉਜ਼ਬੇਕਿਸਤਾਨੀ ਸੋਮ",
             "pl": "Sum",
             "pt": "som usbeque",
             "ro": "Som uzbec",
             "ru": "узбекский сум",
+            "sk": "Uzbecký som",
             "sr": "узбекистански сом",
             "sv": "Uzbekistansk som",
             "tr": "Özbekistan somu",
@@ -5645,7 +5659,7 @@
             "eo": "specialaj rajtoj de enspezo",
             "es": "Derechos Especiales de Giro",
             "eu": "igorpen eskubide bereziak",
-            "fi": "Erityisnosto-oikeus",
+            "fi": "erityisnosto-oikeus",
             "fr": "droits de tirage spéciaux",
             "hr": "Posebna prava vučenja",
             "hu": "különleges lehívási jog",
@@ -5655,6 +5669,7 @@
             "ko": "특별인출권",
             "lt": "Specialiosios skolinimosi teisės",
             "lv": "Speciālās aizņēmuma tiesības",
+            "ms": "hak pengeluaran khas",
             "nl": "speciale trekkingsrechten",
             "oc": "Drechs de tiratge Especials",
             "pl": "specjalne prawa ciągnienia",
@@ -5837,7 +5852,7 @@
             "lt": "Randas",
             "lv": "Dienvidāfrikas rands",
             "ml": "സൗത്ത് ആഫ്രിക്കൻ റാൻഡ്",
-            "ms": "Rand",
+            "ms": "Rand Afrika Selatan",
             "nl": "Zuid-Afrikaanse rand",
             "oc": "Rand sudafrican",
             "pl": "Rand",
@@ -5900,6 +5915,7 @@
             "ko": "짐바브웨 골드",
             "nl": "Zimbabwe Gold",
             "pl": "Złoto Zimbabwe",
+            "pt": "Ouro do Zimbábue",
             "ru": "зимбабвийский золотой",
             "sk": "zimbabwiansky zlatý",
             "sl": "zimbabvejski gold",
@@ -7817,6 +7833,7 @@
         "eritrese nakfa": "ERN",
         "erityinen nosto oikeus": "XDR",
         "erityiset nosto oikeudet": "XDR",
+        "erityisnosto oikeudet": "XDR",
         "erityisnosto oikeus": "XDR",
         "ermeni dramı": "AMD",
         "ermenistan dramı": "AMD",
@@ -8372,6 +8389,8 @@
         "haitský gourde": "HTG",
         "haïtiaanse gourde": "HTG",
         "hak penarikan khusus": "XDR",
+        "hak pengeluaran khas": "XDR",
+        "hak pengeluaran khusus": "XDR",
         "halalas": "SAR",
         "hegoafrikar rand": "ZAR",
         "heller": "CZK",
@@ -9115,6 +9134,7 @@
         "leu da roménia": "RON",
         "leu da romênia": "RON",
         "leu de moldàvia": "MDL",
+        "leu de moldova": "MDL",
         "leu moldau": "MDL",
         "leu moldave": "MDL",
         "leu moldavo": "MDL",
@@ -9122,6 +9142,7 @@
         "leu moldofa": "MDL",
         "leu moldova": "MDL",
         "leu moldovenesc": "MDL",
+        "leu moldovo": "MDL",
         "leu romanès": "RON",
         "leu romanés": "RON",
         "leu romanian": "RON",
@@ -9437,6 +9458,7 @@
         "manat azerbaijandar": "AZN",
         "manat azerbaijanês": "AZN",
         "manat azerbaijano": "AZN",
+        "manat azerbaitjanés": "AZN",
         "manat azerbaiyano": "AZN",
         "manat azerbaïdjanais": "AZN",
         "manat azerbejdżański": "AZN",
@@ -9520,6 +9542,7 @@
         "mauritanijska ouguja": "MRU",
         "mauritanijska uguija": "MRU",
         "mauritániai ouguiya": "MRU",
+        "mauritánska ukíjá": "MRU",
         "mauritánská ukíjá": "MRU",
         "mauritānijas oguja": "MRU",
         "mauritiaanse roepee": "MUR",
@@ -9985,6 +10008,7 @@
         "ouguiya mauritana": "MRU",
         "ouguiya mauritanien": "MRU",
         "ouguiya mawritania": "MRU",
+        "ouro do zimbábue": "ZWG",
         "örmény dram": "AMD",
         "östkaribisk dollar": "XCD",
         "özbekistan somu": "UZS",
@@ -10796,6 +10820,7 @@
         "salomona dolaro": "SBD",
         "salomondollar": "SBD",
         "salomonen dollar": "SBD",
+        "salomoninsaarten dollari": "SBD",
         "salomonsaarten dollari": "SBD",
         "salomonskootočni dolar": "SBD",
         "salüng": "THB",
@@ -11152,6 +11177,7 @@
         "srilankansk rupee": "LKR",
         "srilankanske rupee": "LKR",
         "srí lanka i rúpia": "LKR",
+        "srílanská rupia": "LKR",
         "srílanská rupie": "LKR",
         "srpski dinar": "RSD",
         "ssp": "SSP",
@@ -11415,6 +11441,7 @@
         "tengue": "KZT",
         "tengue cazaque": "KZT",
         "teňňe": "TMT",
+        "tetri": "GEL",
         "thai baht": "THB",
         "thai bát": "THB",
         "thailandiar baht": "THB",
@@ -11539,10 +11566,10 @@
         "turkisk lira": "TRY",
         "turkiska lira": "TRY",
         "turkmeense manat": "TMT",
+        "turkmen manat": "TMT",
         "turkmena manato": "TMT",
         "turkmenistan manat": "TMT",
         "turkmenistan new manat": "TMT",
-        "turkmenistani manat": "TMT",
         "turkmenistani new manat": "TMT",
         "turkmenistanin manat": "TMT",
         "turkmenistansk manat": "TMT",
@@ -11708,6 +11735,7 @@
         "uzbekistano sumas": "UZS",
         "uzbekistansk som": "UZS",
         "uzbekistanski som": "UZS",
+        "uzbekistānas soms": "UZS",
         "uzs": "UZS",
         "új zélandi dollár": "NZD",
         "ürdün dinarı": "JOD",
@@ -11861,6 +11889,7 @@
         "yuan cinese": "CNY",
         "yuan renmimbi": "CNY",
         "yuan renminbi": "CNY",
+        "yuan rmb": "CNY",
         "yuans": "CNY",
         "yuán chino": "CNY",
         "z$": "ZWL",
@@ -13734,6 +13763,7 @@
         "دينار بحريني": "BHD",
         "دينار تونسي": "TND",
         "دينار جزائري": "DZD",
+        "دينار ذهبي": "LYD",
         "دينار سوداني": "SDG",
         "دينار صربي": "RSD",
         "دينار عراقي": "IQD",
@@ -14326,6 +14356,7 @@
             "USD",
             "TWD"
         ],
+        "ดอลลาร์นิวซีแลนด์": "NZD",
         "ดอลลาร์บรูไน": "BND",
         "ดอลลาร์สหรัฐ": "USD",
         "ดอลลาร์สิงคโปร์": "SGD",
@@ -14998,6 +15029,7 @@
         "ボツワナ・プラ": "BWP",
         "ボリバル・ソベラノ": "VES",
         "ボリビアーノ": "BOB",
+        "ポンド・スターリング": "GBP",
         "ポーランド・ズウォティ": [
             "PLZ",
             "PLN"
@@ -15063,7 +15095,6 @@
         "中華人民共和国の通貨": "CNY",
         "中部アフリカcfaフラン": "XAF",
         "人民元": "CNY",
-        "人民币": "CNY",
         "人民幣": "CNY",
         "元": [
             "HKD",

searx/data/engine_traits.json

@@ -180,7 +180,7 @@
       "es": "es-es",
       "et": "et-et",
       "eu": "eu-eu",
-      "fa": "prs-prs",
+      "fa": "fa-fa",
       "fi": "fi-fi",
       "fil": "fil-fil",
       "fr": "fr-fr",
@@ -188,14 +188,12 @@
       "gd": "gd-gd",
       "gl": "gl-gl",
       "gu": "gu-gu",
-      "ha": "ha-latn",
       "he": "he-he",
       "hi": "hi-hi",
       "hr": "hr-hr",
       "hu": "hu-hu",
       "hy": "hy-hy",
       "id": "id-id",
-      "ig": "ig-ig",
       "is": "is-is",
       "it": "it-it",
       "ja": "ja-ja",
@@ -205,8 +203,6 @@
       "kn": "kn-kn",
       "ko": "ko-ko",
       "kok": "kok-kok",
-      "ku": "ku-arab",
-      "ky": "ky-ky",
       "lb": "lb-lb",
       "lo": "lo-lo",
       "lt": "lt-lt",
@@ -214,7 +210,6 @@
       "mi": "mi-mi",
       "mk": "mk-mk",
       "ml": "ml-ml",
-      "mn": "mn-cyrl-mn",
       "mr": "mr-mr",
       "ms": "ms-ms",
       "mt": "mt-mt",
@@ -222,33 +217,22 @@
       "ne": "ne-ne",
       "nl": "nl-nl",
       "nn": "nn-nn",
-      "nso": "nso-nso",
       "or": "or-or",
-      "pa_Arab": "pa-arab",
       "pa_Guru": "pa-guru",
       "pl": "pl-pl",
       "pt": "pt-br",
       "qu": "quz-quz",
-      "quc": "quc-quc",
       "ro": "ro-ro",
       "ru": "ru-ru",
-      "rw": "rw-rw",
-      "sd_Arab": "sd-arab",
-      "si": "si-si",
       "sk": "sk-sk",
       "sl": "sl-sl",
       "sq": "sq-sq",
       "sr_Cyrl": "sr-cyrl",
       "sr_Latn": "sr-latn",
       "sv": "sv-sv",
-      "sw": "sw-sw",
       "ta": "ta-ta",
       "te": "te-te",
-      "tg": "tg-cyrl",
       "th": "th-th",
-      "ti": "ti-ti",
-      "tk": "tk-tk",
-      "tn": "tn-tn",
       "tr": "tr-tr",
       "tt": "tt-tt",
       "ug": "ug-ug",
@@ -256,13 +240,9 @@
       "ur": "ur-ur",
       "uz_Latn": "uz-latn",
       "vi": "vi-vi",
-      "wo": "wo-wo",
-      "xh": "xh-xh",
-      "yo": "yo-yo",
       "zh": "zh-hans",
       "zh_Hans": "zh-hans",
-      "zh_Hant": "zh-hant",
+      "zh_Hant": "zh-hant"
-      "zu": "zu-zu"
     },
     "regions": {
       "am-ET": "am-et",
@@ -478,14 +458,12 @@
       "kk-KZ": "kk-kz",
       "km-KH": "km-kh",
       "ko-KR": "ko-kr",
-      "ky-KG": "ky-kg",
       "lb-LU": "lb-lu",
       "lo-LA": "lo-la",
       "lt-LT": "lt-lt",
       "lv-LV": "lv-lv",
       "mi-NZ": "mi-nz",
       "mk-MK": "mk-mk",
-      "mn-MN": "mn-mn",
       "ms-BN": "ms-bn",
       "ms-MY": "ms-my",
       "ms-SG": "ms-sg",
@@ -519,8 +497,6 @@
       "ru-KZ": "ru-kz",
       "ru-RU": "ru-ru",
       "ru-UA": "ru-ua",
-      "rw-RW": "rw-rw",
-      "si-LK": "si-lk",
       "sk-SK": "sk-sk",
       "sl-SI": "sl-si",
       "sq-AL": "sq-al",
@@ -529,23 +505,14 @@
       "sr-RS": "sr-rs",
       "sv-FI": "sv-fi",
       "sv-SE": "sv-se",
-      "sw-KE": "sw-ke",
-      "sw-TZ": "sw-tz",
-      "sw-UG": "sw-ug",
       "ta-LK": "ta-lk",
       "ta-SG": "ta-sg",
-      "tg-TJ": "tg-tj",
       "th-TH": "th-th",
-      "ti-ER": "ti-er",
-      "tk-TM": "tk-tm",
-      "tn-BW": "tn-bw",
       "tr-CY": "tr-cy",
       "tr-TR": "tr-tr",
       "uk-UA": "uk-ua",
       "ur-PK": "ur-pk",
       "vi-VN": "vi-vn",
-      "wo-SN": "wo-sn",
-      "yo-NG": "yo-ng",
       "zh-CN": "zh-cn",
       "zh-HK": "en-hk",
       "zh-MO": "zh-mo",
@@ -578,7 +545,7 @@
       "es": "es-es",
       "et": "et-et",
       "eu": "eu-eu",
-      "fa": "prs-prs",
+      "fa": "fa-fa",
       "fi": "fi-fi",
       "fil": "fil-fil",
       "fr": "fr-fr",
@@ -586,14 +553,12 @@
       "gd": "gd-gd",
       "gl": "gl-gl",
       "gu": "gu-gu",
-      "ha": "ha-latn",
       "he": "he-he",
       "hi": "hi-hi",
       "hr": "hr-hr",
       "hu": "hu-hu",
       "hy": "hy-hy",
       "id": "id-id",
-      "ig": "ig-ig",
       "is": "is-is",
       "it": "it-it",
       "ja": "ja-ja",
@@ -603,8 +568,6 @@
       "kn": "kn-kn",
       "ko": "ko-ko",
       "kok": "kok-kok",
-      "ku": "ku-arab",
-      "ky": "ky-ky",
       "lb": "lb-lb",
       "lo": "lo-lo",
       "lt": "lt-lt",
@@ -612,7 +575,6 @@
       "mi": "mi-mi",
       "mk": "mk-mk",
       "ml": "ml-ml",
-      "mn": "mn-cyrl-mn",
       "mr": "mr-mr",
       "ms": "ms-ms",
       "mt": "mt-mt",
@@ -620,33 +582,22 @@
       "ne": "ne-ne",
       "nl": "nl-nl",
       "nn": "nn-nn",
-      "nso": "nso-nso",
       "or": "or-or",
-      "pa_Arab": "pa-arab",
       "pa_Guru": "pa-guru",
       "pl": "pl-pl",
       "pt": "pt-br",
       "qu": "quz-quz",
-      "quc": "quc-quc",
       "ro": "ro-ro",
       "ru": "ru-ru",
-      "rw": "rw-rw",
-      "sd_Arab": "sd-arab",
-      "si": "si-si",
       "sk": "sk-sk",
       "sl": "sl-sl",
       "sq": "sq-sq",
       "sr_Cyrl": "sr-cyrl",
       "sr_Latn": "sr-latn",
       "sv": "sv-sv",
-      "sw": "sw-sw",
       "ta": "ta-ta",
       "te": "te-te",
-      "tg": "tg-cyrl",
       "th": "th-th",
-      "ti": "ti-ti",
-      "tk": "tk-tk",
-      "tn": "tn-tn",
       "tr": "tr-tr",
       "tt": "tt-tt",
       "ug": "ug-ug",
@@ -654,13 +605,9 @@
       "ur": "ur-ur",
       "uz_Latn": "uz-latn",
       "vi": "vi-vi",
-      "wo": "wo-wo",
-      "xh": "xh-xh",
-      "yo": "yo-yo",
       "zh": "zh-hans",
       "zh_Hans": "zh-hans",
-      "zh_Hant": "zh-hant",
+      "zh_Hant": "zh-hant"
-      "zu": "zu-zu"
     },
     "regions": {
       "am-ET": "am-et",
@@ -876,14 +823,12 @@
       "kk-KZ": "kk-kz",
       "km-KH": "km-kh",
       "ko-KR": "ko-kr",
-      "ky-KG": "ky-kg",
       "lb-LU": "lb-lu",
       "lo-LA": "lo-la",
       "lt-LT": "lt-lt",
       "lv-LV": "lv-lv",
       "mi-NZ": "mi-nz",
       "mk-MK": "mk-mk",
-      "mn-MN": "mn-mn",
       "ms-BN": "ms-bn",
       "ms-MY": "ms-my",
       "ms-SG": "ms-sg",
@@ -917,8 +862,6 @@
       "ru-KZ": "ru-kz",
       "ru-RU": "ru-ru",
       "ru-UA": "ru-ua",
-      "rw-RW": "rw-rw",
-      "si-LK": "si-lk",
       "sk-SK": "sk-sk",
       "sl-SI": "sl-si",
       "sq-AL": "sq-al",
@@ -927,23 +870,14 @@
       "sr-RS": "sr-rs",
       "sv-FI": "sv-fi",
       "sv-SE": "sv-se",
-      "sw-KE": "sw-ke",
-      "sw-TZ": "sw-tz",
-      "sw-UG": "sw-ug",
       "ta-LK": "ta-lk",
       "ta-SG": "ta-sg",
-      "tg-TJ": "tg-tj",
       "th-TH": "th-th",
-      "ti-ER": "ti-er",
-      "tk-TM": "tk-tm",
-      "tn-BW": "tn-bw",
       "tr-CY": "tr-cy",
       "tr-TR": "tr-tr",
       "uk-UA": "uk-ua",
       "ur-PK": "ur-pk",
       "vi-VN": "vi-vn",
-      "wo-SN": "wo-sn",
-      "yo-NG": "yo-ng",
       "zh-CN": "zh-cn",
       "zh-HK": "en-hk",
       "zh-MO": "zh-mo",
@@ -976,7 +910,7 @@
       "es": "es-es",
       "et": "et-et",
       "eu": "eu-eu",
-      "fa": "prs-prs",
+      "fa": "fa-fa",
       "fi": "fi-fi",
       "fil": "fil-fil",
       "fr": "fr-fr",
@@ -984,14 +918,12 @@
       "gd": "gd-gd",
       "gl": "gl-gl",
       "gu": "gu-gu",
-      "ha": "ha-latn",
       "he": "he-he",
       "hi": "hi-hi",
       "hr": "hr-hr",
       "hu": "hu-hu",
       "hy": "hy-hy",
       "id": "id-id",
-      "ig": "ig-ig",
       "is": "is-is",
       "it": "it-it",
       "ja": "ja-ja",
@@ -1001,8 +933,6 @@
       "kn": "kn-kn",
       "ko": "ko-ko",
       "kok": "kok-kok",
-      "ku": "ku-arab",
-      "ky": "ky-ky",
       "lb": "lb-lb",
       "lo": "lo-lo",
       "lt": "lt-lt",
@@ -1010,7 +940,6 @@
       "mi": "mi-mi",
       "mk": "mk-mk",
       "ml": "ml-ml",
-      "mn": "mn-cyrl-mn",
       "mr": "mr-mr",
       "ms": "ms-ms",
       "mt": "mt-mt",
@@ -1018,33 +947,22 @@
       "ne": "ne-ne",
       "nl": "nl-nl",
       "nn": "nn-nn",
-      "nso": "nso-nso",
       "or": "or-or",
-      "pa_Arab": "pa-arab",
       "pa_Guru": "pa-guru",
       "pl": "pl-pl",
       "pt": "pt-br",
       "qu": "quz-quz",
-      "quc": "quc-quc",
       "ro": "ro-ro",
       "ru": "ru-ru",
-      "rw": "rw-rw",
-      "sd_Arab": "sd-arab",
-      "si": "si-si",
       "sk": "sk-sk",
       "sl": "sl-sl",
       "sq": "sq-sq",
       "sr_Cyrl": "sr-cyrl",
       "sr_Latn": "sr-latn",
       "sv": "sv-sv",
-      "sw": "sw-sw",
       "ta": "ta-ta",
       "te": "te-te",
-      "tg": "tg-cyrl",
       "th": "th-th",
-      "ti": "ti-ti",
-      "tk": "tk-tk",
-      "tn": "tn-tn",
       "tr": "tr-tr",
       "tt": "tt-tt",
       "ug": "ug-ug",
@@ -1052,13 +970,9 @@
       "ur": "ur-ur",
       "uz_Latn": "uz-latn",
       "vi": "vi-vi",
-      "wo": "wo-wo",
-      "xh": "xh-xh",
-      "yo": "yo-yo",
       "zh": "zh-hans",
       "zh_Hans": "zh-hans",
-      "zh_Hant": "zh-hant",
+      "zh_Hant": "zh-hant"
-      "zu": "zu-zu"
     },
     "regions": {
       "am-ET": "am-et",
@@ -1274,14 +1188,12 @@
       "kk-KZ": "kk-kz",
       "km-KH": "km-kh",
       "ko-KR": "ko-kr",
-      "ky-KG": "ky-kg",
       "lb-LU": "lb-lu",
       "lo-LA": "lo-la",
       "lt-LT": "lt-lt",
       "lv-LV": "lv-lv",
       "mi-NZ": "mi-nz",
       "mk-MK": "mk-mk",
-      "mn-MN": "mn-mn",
       "ms-BN": "ms-bn",
       "ms-MY": "ms-my",
       "ms-SG": "ms-sg",
@@ -1315,8 +1227,6 @@
       "ru-KZ": "ru-kz",
       "ru-RU": "ru-ru",
       "ru-UA": "ru-ua",
-      "rw-RW": "rw-rw",
-      "si-LK": "si-lk",
       "sk-SK": "sk-sk",
       "sl-SI": "sl-si",
       "sq-AL": "sq-al",
@@ -1325,23 +1235,14 @@
       "sr-RS": "sr-rs",
       "sv-FI": "sv-fi",
       "sv-SE": "sv-se",
-      "sw-KE": "sw-ke",
-      "sw-TZ": "sw-tz",
-      "sw-UG": "sw-ug",
       "ta-LK": "ta-lk",
       "ta-SG": "ta-sg",
-      "tg-TJ": "tg-tj",
       "th-TH": "th-th",
-      "ti-ER": "ti-er",
-      "tk-TM": "tk-tm",
-      "tn-BW": "tn-bw",
       "tr-CY": "tr-cy",
       "tr-TR": "tr-tr",
       "uk-UA": "uk-ua",
       "ur-PK": "ur-pk",
       "vi-VN": "vi-vn",
-      "wo-SN": "wo-sn",
-      "yo-NG": "yo-ng",
       "zh-CN": "en-hk",
       "zh-HK": "en-hk",
       "zh-MO": "zh-mo",
@@ -1374,7 +1275,7 @@
       "es": "es-es",
       "et": "et-et",
       "eu": "eu-eu",
-      "fa": "prs-prs",
+      "fa": "fa-fa",
       "fi": "fi-fi",
       "fil": "fil-fil",
       "fr": "fr-fr",
@@ -1382,14 +1283,12 @@
       "gd": "gd-gd",
       "gl": "gl-gl",
       "gu": "gu-gu",
-      "ha": "ha-latn",
       "he": "he-he",
       "hi": "hi-hi",
       "hr": "hr-hr",
       "hu": "hu-hu",
       "hy": "hy-hy",
       "id": "id-id",
-      "ig": "ig-ig",
       "is": "is-is",
       "it": "it-it",
       "ja": "ja-ja",
@@ -1399,8 +1298,6 @@
       "kn": "kn-kn",
       "ko": "ko-ko",
       "kok": "kok-kok",
-      "ku": "ku-arab",
-      "ky": "ky-ky",
       "lb": "lb-lb",
       "lo": "lo-lo",
       "lt": "lt-lt",
@@ -1408,7 +1305,6 @@
       "mi": "mi-mi",
       "mk": "mk-mk",
       "ml": "ml-ml",
-      "mn": "mn-cyrl-mn",
       "mr": "mr-mr",
       "ms": "ms-ms",
       "mt": "mt-mt",
@@ -1416,33 +1312,22 @@
       "ne": "ne-ne",
       "nl": "nl-nl",
       "nn": "nn-nn",
-      "nso": "nso-nso",
       "or": "or-or",
-      "pa_Arab": "pa-arab",
       "pa_Guru": "pa-guru",
       "pl": "pl-pl",
       "pt": "pt-br",
       "qu": "quz-quz",
-      "quc": "quc-quc",
       "ro": "ro-ro",
       "ru": "ru-ru",
-      "rw": "rw-rw",
-      "sd_Arab": "sd-arab",
-      "si": "si-si",
       "sk": "sk-sk",
       "sl": "sl-sl",
       "sq": "sq-sq",
       "sr_Cyrl": "sr-cyrl",
       "sr_Latn": "sr-latn",
       "sv": "sv-sv",
-      "sw": "sw-sw",
       "ta": "ta-ta",
       "te": "te-te",
-      "tg": "tg-cyrl",
       "th": "th-th",
-      "ti": "ti-ti",
-      "tk": "tk-tk",
-      "tn": "tn-tn",
       "tr": "tr-tr",
       "tt": "tt-tt",
       "ug": "ug-ug",
@@ -1450,13 +1335,9 @@
       "ur": "ur-ur",
       "uz_Latn": "uz-latn",
       "vi": "vi-vi",
-      "wo": "wo-wo",
-      "xh": "xh-xh",
-      "yo": "yo-yo",
       "zh": "zh-hans",
       "zh_Hans": "zh-hans",
-      "zh_Hant": "zh-hant",
+      "zh_Hant": "zh-hant"
-      "zu": "zu-zu"
     },
     "regions": {
       "am-ET": "am-et",
@@ -1672,14 +1553,12 @@
       "kk-KZ": "kk-kz",
       "km-KH": "km-kh",
       "ko-KR": "ko-kr",
-      "ky-KG": "ky-kg",
       "lb-LU": "lb-lu",
       "lo-LA": "lo-la",
       "lt-LT": "lt-lt",
       "lv-LV": "lv-lv",
       "mi-NZ": "mi-nz",
       "mk-MK": "mk-mk",
-      "mn-MN": "mn-mn",
       "ms-BN": "ms-bn",
       "ms-MY": "ms-my",
       "ms-SG": "ms-sg",
@@ -1713,8 +1592,6 @@
       "ru-KZ": "ru-kz",
       "ru-RU": "ru-ru",
       "ru-UA": "ru-ua",
-      "rw-RW": "rw-rw",
-      "si-LK": "si-lk",
       "sk-SK": "sk-sk",
       "sl-SI": "sl-si",
       "sq-AL": "sq-al",
@@ -1723,23 +1600,14 @@
       "sr-RS": "sr-rs",
       "sv-FI": "sv-fi",
       "sv-SE": "sv-se",
-      "sw-KE": "sw-ke",
-      "sw-TZ": "sw-tz",
-      "sw-UG": "sw-ug",
       "ta-LK": "ta-lk",
       "ta-SG": "ta-sg",
-      "tg-TJ": "tg-tj",
       "th-TH": "th-th",
-      "ti-ER": "ti-er",
-      "tk-TM": "tk-tm",
-      "tn-BW": "tn-bw",
       "tr-CY": "tr-cy",
       "tr-TR": "tr-tr",
       "uk-UA": "uk-ua",
       "ur-PK": "ur-pk",
       "vi-VN": "vi-vn",
-      "wo-SN": "wo-sn",
-      "yo-NG": "yo-ng",
       "zh-CN": "zh-cn",
       "zh-HK": "en-hk",
       "zh-MO": "zh-mo",
@@ -1759,6 +1627,7 @@
         "el": "el",
         "en-CA": "en-ca",
         "en-GB": "en-gb",
+        "en-IN": "en-in",
         "en-US": "en-us",
         "es": "es",
         "et": "et",
@@ -1769,7 +1638,6 @@
         "hu": "hu",
         "id": "id",
         "it": "it",
-        "ja-JP": "ja-jp",
         "ko": "ko",
         "lt": "lt",
         "lv": "lv",
@@ -1860,6 +1728,7 @@
         "el": "el",
         "en-CA": "en-ca",
         "en-GB": "en-gb",
+        "en-IN": "en-in",
         "en-US": "en-us",
         "es": "es",
         "et": "et",
@@ -1870,7 +1739,6 @@
         "hu": "hu",
         "id": "id",
         "it": "it",
-        "ja-JP": "ja-jp",
         "ko": "ko",
         "lt": "lt",
         "lv": "lv",
@@ -1961,6 +1829,7 @@
         "el": "el",
         "en-CA": "en-ca",
         "en-GB": "en-gb",
+        "en-IN": "en-in",
         "en-US": "en-us",
         "es": "es",
         "et": "et",
@@ -1971,7 +1840,6 @@
         "hu": "hu",
         "id": "id",
         "it": "it",
-        "ja-JP": "ja-jp",
         "ko": "ko",
         "lt": "lt",
         "lv": "lv",
@@ -2062,6 +1930,7 @@
         "el": "el",
         "en-CA": "en-ca",
         "en-GB": "en-gb",
+        "en-IN": "en-in",
         "en-US": "en-us",
         "es": "es",
         "et": "et",
@@ -2072,7 +1941,6 @@
         "hu": "hu",
         "id": "id",
         "it": "it",
-        "ja-JP": "ja-jp",
         "ko": "ko",
         "lt": "lt",
         "lv": "lv",
@@ -7609,6 +7477,7 @@
         "ami",
         "an",
         "ang",
+        "ann",
         "anp",
         "ar",
         "arc",
@@ -7727,6 +7596,7 @@
         "hy",
         "hyw",
         "ia",
+        "iba",
         "id",
         "ie",
         "ig",
@@ -7815,6 +7685,7 @@
         "no",
         "nov",
         "nqo",
+        "nr",
         "nrm",
         "nso",
         "nv",
@@ -7848,6 +7719,7 @@
         "ro",
         "roa-rup",
         "roa-tara",
+        "rsk",
         "ru",
         "rue",
         "rw",
@@ -7886,6 +7758,7 @@
         "ta",
         "tay",
         "tcy",
+        "tdd",
         "te",
         "tet",
         "tg",

searx/data/useragents.json

@@ -5,7 +5,7 @@
     ],
     "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
     "versions": [
-        "130.0",
+        "132.0",
-        "129.0"
+        "131.0"
     ]
 }

searx/data/wikidata_units.json

@@ -832,7 +832,7 @@
     "Q104907390": {
         "si_name": "Q182429",
         "symbol": "nmi/h",
-        "to_si_factor": 0.514444
+        "to_si_factor": 0.5144444444444445
     },
     "Q104907398": {
         "si_name": "Q215571",
@@ -1336,7 +1336,7 @@
     },
     "Q106636307": {
         "si_name": "Q80842107",
-        "symbol": "μS/cm",
+        "symbol": "μS/cm-1",
         "to_si_factor": 0.0001
     },
     "Q106639711": {
@@ -4142,7 +4142,7 @@
     "Q23931103": {
         "si_name": "Q25343",
         "symbol": "nmi²",
-        "to_si_factor": 3434290.0120544
+        "to_si_factor": 3429904.0
     },
     "Q239830": {
         "si_name": "Q3395194",

searx/engines/duckduckgo.py

@@ -18,7 +18,6 @@ from searx import (
 )
 from searx.utils import (
     eval_xpath,
-    eval_xpath_getindex,
     extract_text,
 )
 from searx.network import get  # see https://github.com/searxng/searxng/issues/762
@@ -54,31 +53,33 @@ paging = True
 time_range_support = True
 safesearch = True  # user can't select but the results are filtered
 
-url = 'https://lite.duckduckgo.com/lite/'
+url = "https://html.duckduckgo.com/html"
-# url_ping = 'https://duckduckgo.com/t/sl_l'
 
 time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
 form_data = {'v': 'l', 'api': 'd.js', 'o': 'json'}
+__CACHE = []
 
 
-def cache_vqd(query, value):
+def _cache_key(data: dict):
+    return 'SearXNG_ddg_web_vqd' + redislib.secret_hash(f"{data['q']}//{data['kl']}")
+
+
+def cache_vqd(data: dict, value):
     """Caches a ``vqd`` value from a query."""
     c = redisdb.client()
     if c:
         logger.debug("cache vqd value: %s", value)
-        key = 'SearXNG_ddg_web_vqd' + redislib.secret_hash(query)
+        c.set(_cache_key(data), value, ex=600)
-        c.set(key, value, ex=600)
+
+    else:
+        logger.debug("MEM cache vqd value: %s", value)
+        if len(__CACHE) > 100:  # cache vqd from last 100 queries
+            __CACHE.pop(0)
+        __CACHE.append((_cache_key(data), value))
 
 
-def get_vqd(query):
+def get_vqd(data):
-    """Returns the ``vqd`` that fits to the *query*.  If there is no ``vqd`` cached
+    """Returns the ``vqd`` that fits to the *query* (``data`` from HTTP POST).
-    (:py:obj:`cache_vqd`) the query is sent to DDG to get a vqd value from the
-    response.
-
-    .. hint::
-
-       If an empty string is returned there are no results for the ``query`` and
-       therefore no ``vqd`` value.
 
     DDG's bot detection is sensitive to the ``vqd`` value.  For some search terms
     (such as extremely long search terms that are often sent by bots), no ``vqd``
@@ -106,28 +107,23 @@ def get_vqd(query):
     - DuckDuckGo News: ``https://duckduckgo.com/news.js??q=...&vqd=...``
 
     """
+
+    key = _cache_key(data)
     value = None
     c = redisdb.client()
     if c:
-        key = 'SearXNG_ddg_web_vqd' + redislib.secret_hash(query)
         value = c.get(key)
         if value or value == b'':
             value = value.decode('utf-8')
-            logger.debug("re-use cached vqd value: %s", value)
+            logger.debug("re-use CACHED vqd value: %s", value)
             return value
 
-    query_url = 'https://duckduckgo.com/?' + urlencode({'q': query})
+    else:
-    res = get(query_url)
+        for k, value in __CACHE:
-    doc = lxml.html.fromstring(res.text)
+            if k == key:
-    for script in doc.xpath("//script[@type='text/javascript']"):
+                logger.debug("MEM re-use CACHED vqd value: %s", value)
-        script = script.text
-        if 'vqd="' in script:
-            value = extr(script, 'vqd="', '"')
-            break
-    logger.debug("new vqd value: '%s'", value)
-    if value is not None:
-        cache_vqd(query, value)
                 return value
+    return None
 
 
 def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
@@ -155,9 +151,10 @@ def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
 
     .. hint::
 
-       `DDG-lite <https://lite.duckduckgo.com/lite>`__ does not offer a language
+       `DDG-lite <https://lite.duckduckgo.com/lite>`__ and the *no Javascript*
-       selection to the user, only a region can be selected by the user
+       page https://html.duckduckgo.com/html do not offer a language selection
-       (``eng_region`` from the example above).  DDG-lite stores the selected
+       to the user, only a region can be selected by the user (``eng_region``
+       from the example above).  DDG-lite and *no Javascript* store the selected
        region in a cookie::
 
          params['cookies']['kl'] = eng_region  # 'ar-es'
@@ -241,10 +238,25 @@ def request(query, params):
 
     query = quote_ddg_bangs(query)
 
-    # request needs a vqd argument
+    if len(query) >= 500:
-    vqd = get_vqd(query)
+        # DDG does not accept queries with more than 499 chars
+        params["url"] = None
+        return
 
+    # Advanced search syntax ends in CAPTCHA
+    # https://duckduckgo.com/duckduckgo-help-pages/results/syntax/
+    query = [
+        x.removeprefix("site:").removeprefix("intitle:").removeprefix("inurl:").removeprefix("filetype:")
+        for x in query.split()
+    ]
     eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
+    if eng_region == "wt-wt":
+        # https://html.duckduckgo.com/html sets an empty value for "all".
+        eng_region = ""
+
+    params['data']['kl'] = eng_region
+    params['cookies']['kl'] = eng_region
+
     # eng_lang = get_ddg_lang(traits, params['searxng_locale'])
 
     params['url'] = url
@@ -252,54 +264,82 @@ def request(query, params):
     params['data']['q'] = query
 
     # The API is not documented, so we do some reverse engineering and emulate
-    # what https://lite.duckduckgo.com/lite/ does when you press "next Page"
+    # what https://html.duckduckgo.com/html does when you press "next Page" link
-    # link again and again ..
+    # again and again ..
 
     params['headers']['Content-Type'] = 'application/x-www-form-urlencoded'
-    params['data']['vqd'] = vqd
 
-    # initial page does not have an offset
+    params['headers']['Sec-Fetch-Dest'] = "document"
+    params['headers']['Sec-Fetch-Mode'] = "navigate"  # at least this one is used by ddg's bot detection
+    params['headers']['Sec-Fetch-Site'] = "same-origin"
+    params['headers']['Sec-Fetch-User'] = "?1"
+
+    # Form of the initial search page does have empty values in the form
+    if params['pageno'] == 1:
+
+        params['data']['b'] = ""
+
+    params['data']['df'] = ''
+    if params['time_range'] in time_range_dict:
+
+        params['data']['df'] = time_range_dict[params['time_range']]
+        params['cookies']['df'] = time_range_dict[params['time_range']]
+
     if params['pageno'] == 2:
+
         # second page does have an offset of 20
         offset = (params['pageno'] - 1) * 20
         params['data']['s'] = offset
         params['data']['dc'] = offset + 1
 
     elif params['pageno'] > 2:
+
         # third and following pages do have an offset of 20 + n*50
         offset = 20 + (params['pageno'] - 2) * 50
         params['data']['s'] = offset
         params['data']['dc'] = offset + 1
 
-    # initial page does not have additional data in the input form
     if params['pageno'] > 1:
 
+        # initial page does not have these additional data in the input form
         params['data']['o'] = form_data.get('o', 'json')
         params['data']['api'] = form_data.get('api', 'd.js')
         params['data']['nextParams'] = form_data.get('nextParams', '')
         params['data']['v'] = form_data.get('v', 'l')
-        params['headers']['Referer'] = 'https://lite.duckduckgo.com/'
+        params['headers']['Referer'] = url
 
-    params['data']['kl'] = eng_region
+        # from here on no more params['data'] shuld be set, since this dict is
-    params['cookies']['kl'] = eng_region
+        # needed to get a vqd value from the cache ..
 
-    params['data']['df'] = ''
+        vqd = get_vqd(params['data'])
-    if params['time_range'] in time_range_dict:
+
-        params['data']['df'] = time_range_dict[params['time_range']]
+        # Certain conditions must be met in order to call up one of the
-        params['cookies']['df'] = time_range_dict[params['time_range']]
+        # following pages ...
+
+        if vqd:
+            params['data']['vqd'] = vqd  # follow up pages / requests needs a vqd argument
+        else:
+            # Don't try to call follow up pages without a vqd value.  DDG
+            # recognizes this as a request from a bot.  This lowers the
+            # reputation of the SearXNG IP and DDG starts to activate CAPTCHAs.
+            params["url"] = None
+            return
+
+        if params['searxng_locale'].startswith("zh"):
+            # Some locales (at least China) do not have a "next page" button and ddg
+            # will return a HTTP/2 403 Forbidden for a request of such a page.
+            params["url"] = None
+            return
 
     logger.debug("param data: %s", params['data'])
     logger.debug("param cookies: %s", params['cookies'])
-    return params
 
 
-def detect_ddg_captcha(dom):
+def is_ddg_captcha(dom):
-    """In case of CAPTCHA ddg open its own *not a Robot* dialog and is
+    """In case of CAPTCHA ddg response its own *not a Robot* dialog and is not
-    not redirected to CAPTCHA page.
+    redirected to a CAPTCHA page."""
-    """
+
-    if eval_xpath(dom, "//form[@id='challenge-form']"):
+    return bool(eval_xpath(dom, "//form[@id='challenge-form']"))
-        # set suspend time to zero is OK --> ddg does not block the IP
-        raise SearxEngineCaptchaException(suspended_time=0)
 
 
 def response(resp):
@@ -309,37 +349,34 @@ def response(resp):
 
     results = []
     doc = lxml.html.fromstring(resp.text)
-    detect_ddg_captcha(doc)
 
-    result_table = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table')
+    if is_ddg_captcha(doc):
+        # set suspend time to zero is OK --> ddg does not block the IP
+        raise SearxEngineCaptchaException(suspended_time=0, message=f"CAPTCHA ({resp.search_params['data'].get('kl')})")
 
-    if len(result_table) == 2:
+    form = eval_xpath(doc, '//input[@name="vqd"]/..')
-        # some locales (at least China) does not have a "next page" button and
-        # the layout of the HTML tables is different.
-        result_table = result_table[1]
-    elif not len(result_table) >= 3:
-        # no more results
-        return []
-    else:
-        result_table = result_table[2]
-        # update form data from response
-        form = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table//input/..')
     if len(form):
-
+        # some locales (at least China) does not have a "next page" button
         form = form[0]
-            form_data['v'] = eval_xpath(form, '//input[@name="v"]/@value')[0]
+        form_vqd = eval_xpath(form, '//input[@name="vqd"]/@value')[0]
-            form_data['api'] = eval_xpath(form, '//input[@name="api"]/@value')[0]
-            form_data['o'] = eval_xpath(form, '//input[@name="o"]/@value')[0]
-            logger.debug('form_data: %s', form_data)
 
-    tr_rows = eval_xpath(result_table, './/tr')
+        cache_vqd(resp.search_params["data"], form_vqd)
-    # In the last <tr> is the form of the 'previous/next page' links
-    tr_rows = tr_rows[:-1]
 
-    len_tr_rows = len(tr_rows)
+    # just select "web-result" and ignore results of class "result--ad result--ad--small"
-    offset = 0
+    for div_result in eval_xpath(doc, '//div[@id="links"]/div[contains(@class, "web-result")]'):
 
-    zero_click_info_xpath = '//html/body/form/div/table[2]/tr[2]/td/text()'
+        item = {}
+        title = eval_xpath(div_result, './/h2/a')
+        if not title:
+            # this is the "No results." item in the result list
+            continue
+        item["title"] = extract_text(title)
+        item["url"] = eval_xpath(div_result, './/h2/a/@href')[0]
+        item["content"] = extract_text(eval_xpath(div_result, './/a[contains(@class, "result__snippet")]')[0])
+
+        results.append(item)
+
+    zero_click_info_xpath = '//div[@id="zero_click_abstract"]'
     zero_click = extract_text(eval_xpath(doc, zero_click_info_xpath)).strip()
 
     if zero_click and "Your IP address is" not in zero_click and "Your user agent:" not in zero_click:
@@ -352,33 +389,6 @@ def response(resp):
             }
         )
 
-    while len_tr_rows >= offset + 4:
-
-        # assemble table rows we need to scrap
-        tr_title = tr_rows[offset]
-        tr_content = tr_rows[offset + 1]
-        offset += 4
-
-        # ignore sponsored Adds <tr class="result-sponsored">
-        if tr_content.get('class') == 'result-sponsored':
-            continue
-
-        a_tag = eval_xpath_getindex(tr_title, './/td//a[@class="result-link"]', 0, None)
-        if a_tag is None:
-            continue
-
-        td_content = eval_xpath_getindex(tr_content, './/td[@class="result-snippet"]', 0, None)
-        if td_content is None:
-            continue
-
-        results.append(
-            {
-                'title': a_tag.text_content(),
-                'content': extract_text(td_content),
-                'url': a_tag.get('href'),
-            }
-        )
-
     return results
 
 

searx/favicons/cache.py

@@ -30,7 +30,7 @@ import tempfile
 import time
 import typer
 
-from pydantic import BaseModel
+import msgspec
 
 from searx import sqlitedb
 from searx import logger
@@ -90,7 +90,7 @@ def init(cfg: "FaviconCacheConfig"):
         raise NotImplementedError(f"favicons db_type '{cfg.db_type}' is unknown")
 
 
-class FaviconCacheConfig(BaseModel):
+class FaviconCacheConfig(msgspec.Struct):  # pylint: disable=too-few-public-methods
     """Configuration of the favicon cache."""
 
     db_type: Literal["sqlite", "mem"] = "sqlite"

searx/favicons/config.py

@@ -4,9 +4,8 @@
 from __future__ import annotations
 
 import pathlib
-from pydantic import BaseModel
+import msgspec
 
-from searx.compat import tomllib
 from .cache import FaviconCacheConfig
 from .proxy import FaviconProxyConfig
 
@@ -19,7 +18,7 @@ TOML_CACHE_CFG: dict[str, "FaviconConfig"] = {}
 DEFAULT_CFG_TOML_PATH = pathlib.Path(__file__).parent / "favicons.toml"
 
 
-class FaviconConfig(BaseModel):
+class FaviconConfig(msgspec.Struct):  # pylint: disable=too-few-public-methods
     """The class aggregates configurations of the favicon tools"""
 
     cfg_schema: int
@@ -28,10 +27,10 @@ class FaviconConfig(BaseModel):
     By specifying a version, it is possible to ensure downward compatibility in
     the event of future changes to the configuration schema"""
 
-    cache: FaviconCacheConfig = FaviconCacheConfig()
+    cache: FaviconCacheConfig = FaviconCacheConfig
     """Setup of the :py:obj:`.cache.FaviconCacheConfig`."""
 
-    proxy: FaviconProxyConfig = FaviconProxyConfig()
+    proxy: FaviconProxyConfig = FaviconCacheConfig
     """Setup of the :py:obj:`.proxy.FaviconProxyConfig`."""
 
     @classmethod
@@ -45,18 +44,22 @@ class FaviconConfig(BaseModel):
             return cached
 
         with cfg_file.open("rb") as f:
+            data = f.read()
 
-            cfg = tomllib.load(f)
+        cfg = msgspec.toml.decode(data, type=_FaviconConfig)
-            cfg = cfg.get("favicons", cfg)
+        schema = cfg.favicons.cfg_schema
-
-            schema = cfg.get("cfg_schema")
         if schema != CONFIG_SCHEMA:
             raise ValueError(
                 f"config schema version {CONFIG_SCHEMA} is needed, version {schema} is given in {cfg_file}"
             )
 
-            cfg = cls(**cfg)
+        cfg = cfg.favicons
         if use_cache and cached:
             TOML_CACHE_CFG[str(cfg_file.resolve())] = cfg
 
         return cfg
+
+
+class _FaviconConfig(msgspec.Struct):  # pylint: disable=too-few-public-methods
+    # wrapper struct for root object "favicons."
+    favicons: FaviconConfig

searx/favicons/proxy.py

@@ -12,7 +12,7 @@ import urllib.parse
 
 import flask
 from httpx import HTTPError
-from pydantic import BaseModel
+import msgspec
 
 from searx import get_setting
 
@@ -41,7 +41,7 @@ def _initial_resolver_map():
     return d
 
 
-class FaviconProxyConfig(BaseModel):
+class FaviconProxyConfig(msgspec.Struct):
     """Configuration of the favicon proxy."""
 
     max_age: int = 60 * 60 * 24 * 7  # seven days
@@ -59,7 +59,7 @@ class FaviconProxyConfig(BaseModel):
     outgoing request of the resolver.  By default, the value from
     :ref:`outgoing.request_timeout <settings outgoing>` setting is used."""
 
-    resolver_map: dict[str, str] = _initial_resolver_map()
+    resolver_map: dict[str, str] = msgspec.field(default_factory=_initial_resolver_map)
     """The resolver_map is a key / value dictionary where the key is the name of
     the resolver and the value is the fully qualifying name (fqn) of resolver's
     function (the callable).  The resolvers from the python module

searx/limiter.py

@@ -111,6 +111,7 @@ from searx.botdetection import (
     http_accept_encoding,
     http_accept_language,
     http_user_agent,
+    http_sec_fetch,
     ip_limit,
     ip_lists,
     get_network,
@@ -178,16 +179,17 @@ def filter_request(request: flask.Request) -> werkzeug.Response | None:
         logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg)
         return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429))
 
-    # methods applied on /
+    # methods applied on all requests
 
     for func in [
         http_user_agent,
     ]:
         val = func.filter_request(network, request, cfg)
         if val is not None:
+            logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(flask.request))
             return val
 
-    # methods applied on /search
+    # methods applied on /search requests
 
     if request.path == '/search':
 
@@ -196,12 +198,15 @@ def filter_request(request: flask.Request) -> werkzeug.Response | None:
             http_accept_encoding,
             http_accept_language,
             http_user_agent,
+            http_sec_fetch,
             ip_limit,
         ]:
             val = func.filter_request(network, request, cfg)
             if val is not None:
+                logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(flask.request))
                 return val
-    logger.debug(f"OK {network}: %s", dump_request(flask.request))
+        logger.debug(f"OK: {network}: %s", dump_request(flask.request))
+
     return None