jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2026-09-11 16:56:05 +00:00
Code Issues Projects Releases Wiki Activity

[mod] network: migrate to curl_cffi

Browse Source
This commit is contained in:
vojkovic
2026-09-03 14:19:54 +00:00
committed by Brock Vojkovic
parent 15b0c8ef3a
commit be836e614a
39 changed files with 355 additions and 455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
searx/network/network.py
View File

@@ -13,11 +13,16 @@ import asyncio
import ipaddress
from itertools import cycle
import httpx
from curl_cffi import CurlHttpVersion
from curl_cffi.requests.exceptions import (
ConnectionError as CurlConnectionError,
ProxyError,
RequestException,
)
from searx import logger, sxng_debug
from searx.extended_types import SXNG_Response
from .client import new_client, get_loop, AsyncHTTPTransportNoHttp
from .client import DEFAULT_IMPERSONATE, AsyncClient, new_client, get_loop
from .raise_for_httperror import raise_for_httperror
@@ -48,9 +53,8 @@ class Network:
'enable_http',
'verify',
'enable_http2',
'enable_http3',
'max_connections',
'max_keepalive_connections',
'keepalive_expiry',
'local_addresses',
'proxies',
'using_tor_proxy',
@@ -64,6 +68,7 @@ class Network:
)
_TOR_CHECK_RESULT = {}
_CLIENT_KWARGS = ('verify', 'max_redirects', 'impersonate', 'curl_options', 'enable_http3')
def __init__(
# pylint: disable=too-many-arguments
@@ -71,9 +76,8 @@ class Network:
enable_http: bool = True,
verify: bool = True,
enable_http2: bool = False,
enable_http3: bool = False,
max_connections: int = None, # pyright: ignore[reportArgumentType]
max_keepalive_connections: int = None, # pyright: ignore[reportArgumentType]
keepalive_expiry: float = None, # pyright: ignore[reportArgumentType]
proxies: str | dict[str, str] | None = None,
using_tor_proxy: bool = False,
local_addresses: str | list[str] | None = None,
@@ -86,9 +90,8 @@ class Network:
self.enable_http = enable_http
self.verify = verify
self.enable_http2 = enable_http2
self.enable_http3 = enable_http3
self.max_connections = max_connections
self.max_keepalive_connections = max_keepalive_connections
self.keepalive_expiry = keepalive_expiry
self.proxies = proxies
self.using_tor_proxy = using_tor_proxy
self.local_addresses = local_addresses
@@ -137,7 +140,6 @@ class Network:
def iter_proxies(self) -> Generator[tuple[str, list[str]]]:
if not self.proxies:
return
# https://www.python-httpx.org/compatibility/#proxy-keys
if isinstance(self.proxies, str):
yield 'all://', [self.proxies]
else:
@@ -155,62 +157,73 @@ class Network:
# pylint: disable=stop-iteration-return
yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
async def log_response(self, response: httpx.Response):
_HTTP_VERSION = {
int(CurlHttpVersion.V1_0): "HTTP/1.0",
int(CurlHttpVersion.V1_1): "HTTP/1.1",
int(CurlHttpVersion.V2_0): "HTTP/2",
int(CurlHttpVersion.V2TLS): "HTTP/2",
int(CurlHttpVersion.V2_PRIOR_KNOWLEDGE): "HTTP/2",
int(CurlHttpVersion.V3): "HTTP/3",
int(CurlHttpVersion.V3ONLY): "HTTP/3",
}
async def log_response(self, response: SXNG_Response):
request = response.request
status = f"{response.status_code} {response.reason_phrase}"
response_line = f"{response.http_version} {status}"
http_version = self._HTTP_VERSION.get(response.http_version, str(response.http_version))
status = f"{response.status_code} {response.reason}"
response_line = f"{http_version} {status}"
content_type = response.headers.get("Content-Type")
content_type = f' ({content_type})' if content_type else ''
self._logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}')
method = request.method if request else "?"
url = request.url if request else response.url
self._logger.debug(f'HTTP Request: {method} {url} "{response_line}"{content_type}')
@staticmethod
async def check_tor_proxy(client: httpx.AsyncClient, proxies) -> bool:
async def check_tor_proxy(client: AsyncClient, proxies) -> bool:
if proxies in Network._TOR_CHECK_RESULT:
return Network._TOR_CHECK_RESULT[proxies]
result = True
# ignore client._transport because it is not used with all://
for transport in client._mounts.values(): # pylint: disable=protected-access
if isinstance(transport, AsyncHTTPTransportNoHttp):
continue
if getattr(transport, "_pool") and getattr(
# pylint: disable=protected-access
transport._pool, # type: ignore
"_rdns",
False,
):
continue
if not proxies or not all(url.startswith('socks5h://') for _, url in proxies):
Network._TOR_CHECK_RESULT[proxies] = False
return False
response = await client.get("https://check.torproject.org/api/ip", timeout=60)
if not response.json()["IsTor"]:
result = False
result = bool(response.json()["IsTor"])
Network._TOR_CHECK_RESULT[proxies] = result
return result
async def get_client(self, verify: bool | None = None, max_redirects: int | None = None) -> httpx.AsyncClient:
async def get_client(
self,
verify: bool | None = None,
max_redirects: int | None = None,
impersonate: str | None = None,
curl_options: dict[int, t.Any] | None = None,
enable_http3: bool | None = None,
) -> AsyncClient:
verify = self.verify if verify is None else verify
max_redirects = self.max_redirects if max_redirects is None else max_redirects
impersonate = impersonate or DEFAULT_IMPERSONATE
enable_http3 = self.enable_http3 if enable_http3 is None else enable_http3
local_address = next(self._local_addresses_cycle)
proxies = next(self._proxies_cycle) # is a tuple so it can be part of the key
key = (verify, max_redirects, local_address, proxies)
hook_log_response = self.log_response if sxng_debug else None
curl_key = tuple(sorted((int(k), v) for k, v in (curl_options or {}).items()))
key = (verify, max_redirects, local_address, proxies, impersonate, curl_key, enable_http3)
if key not in self._clients or self._clients[key].is_closed:
client = new_client(
self.enable_http,
verify,
self.enable_http2,
enable_http3,
self.max_connections,
self.max_keepalive_connections,
self.keepalive_expiry,
dict(proxies),
local_address,
0,
max_redirects,
hook_log_response,
impersonate=impersonate,
curl_options=curl_options,
)
if self.using_tor_proxy and not await self.check_tor_proxy(client, proxies):
await client.aclose()
raise httpx.ProxyError('Network configuration problem: not using Tor')
raise ProxyError('Network configuration problem: not using Tor')
self._clients[key] = client
return self._clients[key]
@@ -218,22 +231,14 @@ class Network:
async def close_client(client):
try:
await client.aclose()
except httpx.HTTPError:
except RequestException:
pass
await asyncio.gather(*[close_client(client) for client in self._clients.values()], return_exceptions=False)
@staticmethod
def extract_kwargs_clients(kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
kwargs_clients: dict[str, t.Any] = {}
if 'verify' in kwargs:
kwargs_clients['verify'] = kwargs.pop('verify')
if 'max_redirects' in kwargs:
kwargs_clients['max_redirects'] = kwargs.pop('max_redirects')
if 'allow_redirects' in kwargs:
# see https://github.com/encode/httpx/pull/1808
kwargs['follow_redirects'] = kwargs.pop('allow_redirects')
return kwargs_clients
return {key: kwargs.pop(key) for key in Network._CLIENT_KWARGS if key in kwargs}
@staticmethod
def extract_do_raise_for_httperror(kwargs: dict[str, t.Any]):
@@ -243,23 +248,18 @@ class Network:
del kwargs['raise_for_httperror']
return do_raise_for_httperror
def patch_response(self, response: httpx.Response, do_raise_for_httperror: bool) -> SXNG_Response:
if isinstance(response, httpx.Response):
response = t.cast(SXNG_Response, response)
# requests compatibility (response is not streamed)
# see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses
response.ok = not response.is_error
# raise an exception
if do_raise_for_httperror:
try:
raise_for_httperror(response)
except:
self._logger.warning(f"HTTP Request failed: {response.request.method} {response.request.url}")
raise
def patch_response(self, response: SXNG_Response, do_raise_for_httperror: bool) -> SXNG_Response:
if do_raise_for_httperror:
try:
raise_for_httperror(response)
except:
method = response.request.method if response.request else "?"
url = response.request.url if response.request else response.url
self._logger.warning(f"HTTP Request failed: {method} {url}")
raise
return response
def is_valid_response(self, response: httpx.Response):
def is_valid_response(self, response: SXNG_Response):
# pylint: disable=too-many-boolean-expressions
if (
(self.retry_on_http_error is True and 400 <= response.status_code <= 599)
@@ -276,26 +276,29 @@ class Network:
kwargs_clients = Network.extract_kwargs_clients(kwargs)
while retries >= 0: # pragma: no cover
client = await self.get_client(**kwargs_clients)
cookies = kwargs.pop("cookies", None)
client.cookies = httpx.Cookies(cookies)
try:
method = method.upper()
client.check_url(url)
if stream:
return client.stream(method, url, **kwargs)
response = await client.request(method, url, **kwargs)
if sxng_debug:
await self.log_response(response)
if self.is_valid_response(response) or retries <= 0:
return self.patch_response(response, do_raise_for_httperror)
except httpx.RemoteProtocolError as e:
await client.aclose()
except CurlConnectionError as e:
if not was_disconnected:
# the server has closed the connection:
# try again without decreasing the retries variable & with a new HTTP client
was_disconnected = True
await client.aclose()
self._logger.warning('httpx.RemoteProtocolError: the server has disconnected, retrying')
self._logger.warning('ConnectionError: the server has disconnected, retrying')
continue
if retries <= 0:
raise e
except (httpx.RequestError, httpx.HTTPStatusError) as e:
except RequestException as e:
if retries <= 0:
raise e
retries -= 1
@@ -346,15 +349,12 @@ def initialize(
settings_engines = settings_engines or settings['engines']
settings_outgoing = settings_outgoing or settings['outgoing']
# default parameters for AsyncHTTPTransport
# see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
default_params: dict[str, t.Any] = {
'enable_http': False,
'verify': settings_outgoing['verify'],
'enable_http2': settings_outgoing['enable_http2'],
'enable_http3': False,
'max_connections': settings_outgoing['pool_connections'],
'max_keepalive_connections': settings_outgoing['pool_maxsize'],
'keepalive_expiry': settings_outgoing['keepalive_expiry'],
'local_addresses': settings_outgoing['source_ips'],
'using_tor_proxy': settings_outgoing['using_tor_proxy'],
'proxies': settings_outgoing['proxies'],
@@ -424,9 +424,6 @@ def initialize(
def done():
"""Close all HTTP client
Avoid a warning at exit
See https://github.com/encode/httpx/pull/2026
Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
So Network.aclose is called here using atexit.register
"""