Merge ac09493a91 into 0f9694c90b

searx/engines/duckduckgo_extra.py

@@ -7,14 +7,16 @@ DuckDuckGo Extra (images, videos, news)
 from datetime import datetime
 from typing import TYPE_CHECKING
 from urllib.parse import urlencode
-from searx.utils import get_embeded_stream_url
+from searx.utils import extr, get_embeded_stream_url
 
 from searx.engines.duckduckgo import fetch_traits  # pylint: disable=unused-import
 from searx.engines.duckduckgo import (
+    cache_vqd,
     get_ddg_lang,
     get_vqd,
 )
 from searx.enginelib.traits import EngineTraits
+from searx.network import get
 
 if TYPE_CHECKING:
     import logging
@@ -47,16 +49,34 @@ safesearch_args = {0: '1', 1: None, 2: '1'}
 search_path_map = {'images': 'i', 'videos': 'v', 'news': 'news'}
 
 
+def _extract_vqd(query, region):
+    resp = get(f'https://duckduckgo.com/?q={query}&t=h_&iar=images&iax=images&ia=images')
+    if resp.status_code != 200:
+        return None
+
+    vqd = extr(resp.text, 'vqd="', '"')
+
+    return vqd
+
+
 def request(query, params):
+    eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
+
+    params['data']['kl'] = eng_region
+    params['data']['q'] = query
 
     # request needs a vqd argument
-    vqd = get_vqd(query)
+    vqd = get_vqd(params['data'])
+    if not vqd:
+        vqd = _extract_vqd(query, eng_region)
+        if vqd:
+            cache_vqd(params['data'], vqd)
+
     if not vqd:
         # some search terms do not have results and therefore no vqd value
         params['url'] = None
         return params
 
-    eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
     eng_lang = get_ddg_lang(traits, params['searxng_locale'])
 
     args = {
@@ -85,6 +105,14 @@ def request(query, params):
     logger.debug("cookies: %s", params['cookies'])
 
     params['url'] = f'https://duckduckgo.com/{search_path_map[ddg_category]}.js?{urlencode(args)}'
+    params['headers'] = {
+        'Referer': 'https://duckduckgo.com/',
+        'X-Requested-With': 'XMLHttpRequest',
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:132.0) Gecko/20100101 Firefox/132.0',
+        'Accept': 'application/json, text/javascript, */*; q=0.01',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate, zstd',
+    }
 
     return params