mirror of
https://github.com/searxng/searxng.git
synced 2026-07-17 21:41:24 +00:00
[fix] qwant: engine blocked with captcha
Qwant now requires a `datadome` cookie that it returns in the first search response as a `Set-Cookie`. This cookie has to be sent for all requests, otherwise they will be blocked. This means that now, the first search request is blocked (results in CAPTCHA), and only the subsequent searches work (same happens on the Qwant website for me). However, I don't think it's worth repeating the same search request multiple times very quickly because that also makes us more suspicious.
This commit is contained in:
@@ -51,6 +51,7 @@ from urllib.parse import urlencode
|
|||||||
import babel
|
import babel
|
||||||
from flask_babel import gettext # pyright: ignore[reportUnknownVariableType]
|
from flask_babel import gettext # pyright: ignore[reportUnknownVariableType]
|
||||||
|
|
||||||
|
from searx.enginelib import EngineCache
|
||||||
from searx.enginelib.traits import EngineTraits
|
from searx.enginelib.traits import EngineTraits
|
||||||
from searx.exceptions import (
|
from searx.exceptions import (
|
||||||
SearxEngineAccessDeniedException,
|
SearxEngineAccessDeniedException,
|
||||||
@@ -105,9 +106,19 @@ qwant_news_locales = [
|
|||||||
]
|
]
|
||||||
# fmt: on
|
# fmt: on
|
||||||
|
|
||||||
|
base_url = "https://www.qwant.com"
|
||||||
api_url = "https://api.qwant.com/v3/search/"
|
api_url = "https://api.qwant.com/v3/search/"
|
||||||
"""URL of Qwant's API (JSON)"""
|
"""URL of Qwant's API (JSON)"""
|
||||||
|
|
||||||
|
CACHE: EngineCache
|
||||||
|
"""Cache for storing the ``datadome`` cookie."""
|
||||||
|
|
||||||
|
|
||||||
|
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||||
|
global CACHE # pylint: disable=global-statement
|
||||||
|
CACHE = EngineCache(engine_settings["name"])
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def request(query: str, params: "OnlineParams") -> None:
|
def request(query: str, params: "OnlineParams") -> None:
|
||||||
"""Qwant search request"""
|
"""Qwant search request"""
|
||||||
@@ -129,24 +140,29 @@ def request(query: str, params: "OnlineParams") -> None:
|
|||||||
"tgp": test_group_value,
|
"tgp": test_group_value,
|
||||||
"device": "desktop",
|
"device": "desktop",
|
||||||
"safesearch": params["safesearch"],
|
"safesearch": params["safesearch"],
|
||||||
"display": True,
|
# True would be encoded to "True", instead of "true", which makes the request
|
||||||
"llm": True,
|
# easier to detect and block
|
||||||
|
"displayed": "true",
|
||||||
|
"llm": "true",
|
||||||
}
|
}
|
||||||
|
|
||||||
# shuffle query parameters to be harder to fingerprint
|
|
||||||
args = list(args.items())
|
|
||||||
random.shuffle(args)
|
|
||||||
args = dict(args)
|
|
||||||
|
|
||||||
params["raise_for_httperror"] = False
|
params["raise_for_httperror"] = False
|
||||||
|
|
||||||
params["url"] = f"{api_url}{qwant_categ}?{urlencode(args)}"
|
params["url"] = f"{api_url}{qwant_categ}?{urlencode(args)}"
|
||||||
|
|
||||||
|
params["cookies"]["datadome"] = CACHE.get("datadome")
|
||||||
|
params["headers"].update({"Accept": "application/json", "Referer": f"{base_url}/", "Origin": base_url})
|
||||||
|
|
||||||
|
|
||||||
def response(resp: "SXNG_Response") -> EngineResults:
|
def response(resp: "SXNG_Response") -> EngineResults:
|
||||||
"""Parse results from Qwant's API"""
|
"""Parse results from Qwant's API"""
|
||||||
# pylint: disable=too-many-locals, too-many-branches, too-many-statements
|
# pylint: disable=too-many-locals, too-many-branches, too-many-statements
|
||||||
|
|
||||||
|
# cache datadome cookie - changes on each request
|
||||||
|
datadome = resp.cookies.get("datadome")
|
||||||
|
if datadome:
|
||||||
|
CACHE.set("datadome", datadome)
|
||||||
|
|
||||||
res = EngineResults()
|
res = EngineResults()
|
||||||
|
|
||||||
# Try to load JSON result
|
# Try to load JSON result
|
||||||
@@ -163,8 +179,8 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
|||||||
error_code = data.get("error_code")
|
error_code = data.get("error_code")
|
||||||
if error_code == 24:
|
if error_code == 24:
|
||||||
raise SearxEngineTooManyRequestsException()
|
raise SearxEngineTooManyRequestsException()
|
||||||
if search_results.get("data", {}).get("error_data", {}).get("captchaUrl") is not None:
|
if search_results.get("url") is not None:
|
||||||
raise SearxEngineCaptchaException()
|
raise SearxEngineCaptchaException(suspended_time=0)
|
||||||
if resp.status_code == 403:
|
if resp.status_code == 403:
|
||||||
raise SearxEngineAccessDeniedException()
|
raise SearxEngineAccessDeniedException()
|
||||||
msg = ",".join(data.get("message", ["unknown"]))
|
msg = ",".join(data.get("message", ["unknown"]))
|
||||||
|
|||||||
Reference in New Issue
Block a user