[mod] network: migrate to curl_cffi

searx/engines/__builtins__.pyi

@@ -26,6 +26,7 @@ categories: list[str]
 disabled: bool
 display_error_messages: bool
 enable_http: bool
+enable_http3: bool
 engine_type: str
 inactive: bool
 max_page: int

searx/engines/bing.py

@@ -40,6 +40,7 @@ about: dict[str, t.Any] = {
 # engine dependent config
 categories = ["general", "web"]
 safesearch = True
+enable_http3 = True
 _safesearch_map: dict[int, str] = {
     0: "off",
     1: "moderate",

searx/engines/bing_images.py

@@ -25,6 +25,7 @@ about = {
 # engine dependent config
 categories = ["images", "web"]
 paging = True
+enable_http3 = True
 safesearch = True
 time_range_support = True
 time_map = {

searx/engines/bing_news.py

@@ -33,6 +33,7 @@ categories = ["news"]
 paging = True
 """If go through the pages and there are actually no new results for another
 page, then bing returns the results from the last page again."""
+enable_http3 = True
 
 time_range_support = True
 time_map = {

searx/engines/bing_videos.py

@@ -26,6 +26,7 @@ about = {
 # engine dependent config
 categories = ["videos", "web"]
 paging = True
+enable_http3 = True
 safesearch = True
 time_range_support = True
 

searx/engines/brave.py

@@ -151,6 +151,7 @@ about = {
 
 base_url = "https://search.brave.com/"
 categories = []
+enable_http3 = True
 brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = "search"
 """Brave supports common web-search, videos, images, news, and goggles search.
 

searx/engines/duckduckgo_extra.py

@@ -98,6 +98,7 @@ def request(query: str, params: "OnlineParams") -> None:
     # The vqd value is generated from the query and the UA header. To be able to
     # reuse the vqd value, the UA header must be static.
     headers["User-Agent"] = _HTTP_User_Agent
+    params["impersonate"] = "none"
     vqd = get_vqd(query=query, params=params) or fetch_vqd(query=query, params=params)
 
     headers["Accept"] = "*/*"

searx/engines/google.py

@@ -327,6 +327,7 @@ def google_request(
 
     params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
     params["headers"]["User-Agent"] = random.choice(nokia_useragents)
+    params["impersonate"] = "chrome99_android"
 
 
 def request(query: str, params: "OnlineParams") -> None:

searx/engines/google_cse.py

@@ -30,6 +30,7 @@ about = {
 
 categories = ["general", "web"]
 paging = True
+enable_http3 = True
 max_page = 5
 page_size = 20
 time_range_support = True

searx/engines/google_images.py

@@ -26,6 +26,7 @@ about = {
 # engine dependent config
 categories = ["images", "web"]
 paging = True
+enable_http3 = True
 max_page = 50
 """Google supports up to 50 pages of results, see the `Google max_page discussion`_.
 

searx/engines/google_play.py

@@ -20,6 +20,7 @@ about = {
 }
 
 play_categ = None  # apps|movies
+enable_http3 = True
 base_url = 'https://play.google.com'
 search_url = base_url + "/store/search?{query}&c={play_categ}"
 

searx/engines/google_scholar.py

@@ -27,7 +27,7 @@ import typing as t
 from urllib.parse import urlencode
 from datetime import datetime
 from lxml import html
-import httpx
+from curl_cffi.requests.exceptions import TooManyRedirects
 
 from searx.utils import (
     eval_xpath,
@@ -63,6 +63,7 @@ about = {
 # engine dependent config
 categories = ["science", "scientific publications"]
 paging = True
+enable_http3 = True
 max_page = 50
 """`Google max 50 pages`_
 
@@ -102,7 +103,7 @@ def response(resp: "SXNG_Response") -> EngineResults:  # pylint: disable=too-man
             raise SearxEngineAccessDeniedException(
                 message="google_scholar: unusual traffic detected",
             )
-        raise httpx.TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
+        raise TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
 
     res = EngineResults()
     dom = html.fromstring(resp.text)

searx/engines/yacy.py

@@ -58,7 +58,7 @@ from json import loads
 from urllib.parse import urlencode
 from dateutil import parser
 
-from httpx import DigestAuth
+from curl_cffi import CurlOpt
 
 from searx.utils import html_to_text
 
@@ -141,7 +141,10 @@ def request(query, params):
     params["url"] = f"{_base_url()}/yacysearch.json?{urlencode(args)}"
 
     if http_digest_auth_user and http_digest_auth_pass:
-        params['auth'] = DigestAuth(http_digest_auth_user, http_digest_auth_pass)
+        params['curl_options'] = {
+            CurlOpt.HTTPAUTH: 2,  # CURLAUTH_DIGEST
+            CurlOpt.USERPWD: f"{http_digest_auth_user}:{http_digest_auth_pass}",
+        }
 
     return params
 

searx/engines/yandex.py

@@ -22,6 +22,7 @@ about = {
 # Engine configuration
 categories = []
 paging = True
+enable_http3 = True
 search_type = ""
 
 # Search URL

searx/engines/youtube_api.py

@@ -22,6 +22,7 @@ about = {
 # engine dependent config
 categories = ['videos', 'music']
 paging = False
+enable_http3 = True
 api_key = None
 
 # search-url

searx/engines/youtube_noapi.py

@@ -20,6 +20,7 @@ about = {
 # engine dependent config
 categories = ['videos', 'music']
 paging = True
+enable_http3 = True
 language_support = False
 time_range_support = True