diff --git a/searx/engines/duckduckgo_web.py b/searx/engines/duckduckgo_web.py
index 01f5c6a7c..1ac174238 100644
--- a/searx/engines/duckduckgo_web.py
+++ b/searx/engines/duckduckgo_web.py
@@ -14,11 +14,12 @@ can't build it ourselves and must scrape it from the HTML pages.
"""
import typing as t
+import re
-from urllib.parse import quote_plus
+from urllib.parse import quote_plus, urljoin
from lxml import html
-from searx.utils import html_to_text, gen_useragent, extract_text, eval_xpath
+from searx.utils import html_to_text, extract_text, eval_xpath
from searx.result_types import EngineResults
from searx.enginelib import EngineCache
from searx.network import get
@@ -38,7 +39,6 @@ about = {
# engine dependent config
categories = ["general"]
paging = True
-_HTTP_User_Agent: str = gen_useragent()
base_url = "https://duckduckgo.com"
@@ -73,6 +73,8 @@ def _fetch_first_page_link(
resp = get(
url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
headers=headers,
+ impersonate="firefox",
+ default_headers=False,
timeout=2,
)
@@ -96,6 +98,43 @@ def _cache_key(query: str, pageno: int) -> str:
return f"nextpage_url|{query}|{pageno}"
+def _solve_jsa(resp: "SXNG_Response") -> "SXNG_Response":
+ """Duckduckgo sometimes issues a challenge instead of json."""
+
+ # length that a real browser would report for where the broken snippet is
+ html_len = {
+ "
None:
if len(query) >= 500:
@@ -103,25 +142,15 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = None
return
- headers = params["headers"]
-
- # The vqd value is generated from the query and the UA header. To be able
- # to reuse the vqd value, the UA header must be static.
- headers["User-Agent"] = _HTTP_User_Agent
- headers["Accept"] = "*/*"
- headers["Referer"] = f"{base_url}/"
- headers["Host"] = "duckduckgo.com"
-
- # Sec-Fetch headers are required to not get blocked when sending a Firefox user agent
- headers["Sec-Fetch-Dest"] = "script"
- headers["Sec-Fetch-Mode"] = "no-cors"
- headers["Sec-Fetch-Site"] = "same-site"
+ # firefox TLS only
+ params["impersonate"] = "firefox"
+ params["default_headers"] = False
api_url = ""
if params["pageno"] > 1:
api_url = CACHE.get(_cache_key(query, params["pageno"]))
else:
- api_url = _fetch_first_page_link(query, headers)
+ api_url = _fetch_first_page_link(query, params["headers"])
if not api_url:
params["url"] = None
@@ -129,14 +158,27 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
+ # loads as a script
+ headers = params["headers"]
+ headers["Accept"] = "*/*"
+ headers["Sec-Fetch-Dest"] = "script"
+ headers["Sec-Fetch-Mode"] = "no-cors"
+ headers["Sec-Fetch-Site"] = "same-site"
+ headers["Referer"] = f"{base_url}/"
+
# TODO: support safesearch, timerange and engine traits # pylint:disable=fixme
def response(resp: "SXNG_Response"):
res = EngineResults()
- res_json = resp.json()
- for result in res_json["results"]:
+ # check if ddg returns a challenge
+ # e.g. 'site:github.com searxng'
+ if "let jsa =" in (resp.text or ""):
+ resp = _solve_jsa(resp)
+
+ results = resp.json()["results"]
+ for result in results:
if "u" not in result:
continue
@@ -144,13 +186,13 @@ def response(resp: "SXNG_Response"):
res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
)
- # link to next page
- next_page_path = res_json["results"][-1].get("n")
- if next_page_path:
- CACHE.set(
- _cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
- base_url + next_page_path,
- expire=60 * 60,
- )
+ if results:
+ next_page_path = results[-1].get("n")
+ if next_page_path:
+ CACHE.set(
+ _cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
+ base_url + next_page_path,
+ expire=60 * 60,
+ )
return res
diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py
index 1fe4721c3..5c51fde56 100644
--- a/searx/search/processors/online.py
+++ b/searx/search/processors/online.py
@@ -88,6 +88,9 @@ class HTTPParams(t.TypedDict):
impersonate: t.NotRequired[str]
"""curl_cffi impersonate target. Default: ``chrome``."""
+ default_headers: t.NotRequired[bool]
+ """Ask curl_cffi to skip setting the impersonated browser's default headers."""
+
curl_options: t.NotRequired[dict[int, t.Any]]
"""Any extra libcurl options for the request."""
@@ -169,8 +172,8 @@ class OnlineProcessor(EngineProcessor):
"cookies": params["cookies"],
"auth": params["auth"],
}
- for key in ("curl_options", "impersonate"):
- if params.get(key):
+ for key in ("curl_options", "impersonate", "default_headers"):
+ if params.get(key) is not None:
request_args[key] = params[key]
verify = params.get("verify")