jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2026-09-11 16:56:05 +00:00
Code Issues Projects Releases Wiki Activity

[mod] network: migrate to curl_cffi

Browse Source
This commit is contained in:
vojkovic
2026-09-03 14:19:54 +00:00
committed by Brock Vojkovic
parent 15b0c8ef3a
commit be836e614a
39 changed files with 355 additions and 455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
searx/network/__init__.py
View File

@@ -14,8 +14,7 @@ from timeit import default_timer
from collections.abc import Iterable
from contextlib import contextmanager
import httpx
import anyio
from curl_cffi.requests.exceptions import StreamConsumedError, Timeout
from searx.extended_types import SXNG_Response
from .network import get_network, initialize, check_network_configuration # pylint:disable=cyclic-import
@@ -74,7 +73,6 @@ def _get_timeout(start_time: float, kwargs: t.Any) -> float:
# pylint: disable=too-many-branches
timeout: float | None
# timeout (httpx)
if 'timeout' in kwargs:
timeout = kwargs['timeout']
else:
@@ -105,10 +103,10 @@ def request(method: str, url: str, **kwargs: t.Any) -> SXNG_Response:
try:
return future.result(timeout)
except concurrent.futures.TimeoutError as e:
raise httpx.TimeoutException('Timeout', request=None) from e
raise Timeout('Timeout') from e
def multi_requests(request_list: list["Request"]) -> list[httpx.Response | Exception]:
def multi_requests(request_list: list["Request"]) -> list[SXNG_Response | Exception]:
"""send multiple HTTP requests in parallel. Wait for all requests to finish."""
with _record_http_time() as start_time:
# send the requests
@@ -128,7 +126,7 @@ def multi_requests(request_list: list["Request"]) -> list[httpx.Response | Excep
try:
responses.append(future.result(timeout))
except concurrent.futures.TimeoutError:
responses.append(httpx.TimeoutException('Timeout', request=None))
responses.append(Timeout('Timeout'))
except Exception as e: # pylint: disable=broad-except
responses.append(e)
return responses
@@ -205,14 +203,12 @@ async def stream_chunk_to_queue(network, queue, method: str, url: str, **kwargs:
try:
async with await network.stream(method, url, **kwargs) as response:
queue.put(response)
# aiter_raw: access the raw bytes on the response without applying any HTTP content decoding
# https://www.python-httpx.org/quickstart/#streaming-responses
async for chunk in response.aiter_raw(65536):
async for chunk in response.aiter_content():
if len(chunk) > 0:
queue.put(chunk)
except (httpx.StreamClosed, anyio.ClosedResourceError):
except StreamConsumedError:
# the response was queued before the exception.
# the exception was raised on aiter_raw.
# the exception was raised on aiter_content.
# we do nothing here: in the finally block, None will be queued
# so stream(method, url, **kwargs) generator can stop
pass
@@ -246,22 +242,19 @@ def _close_response_method(self):
asyncio.run_coroutine_threadsafe(self.aclose(), get_loop())
# reach the end of _self.generator ( _stream_generator ) to an avoid memory leak.
# it makes sure that :
# * the httpx response is closed (see the stream_chunk_to_queue function)
# * the curl_cffi response is closed (see the stream_chunk_to_queue function)
# * to call future.result() in _stream_generator
for _ in self._generator: # pylint: disable=protected-access
continue
def stream(method: str, url: str, **kwargs: t.Any) -> tuple[SXNG_Response, Iterable[bytes]]:
"""Replace httpx.stream.
"""Stream for the image proxy.
Usage:
response, stream = poolrequests.stream(...)
response, stream = searx.network.stream(...)
for chunk in stream:
...
httpx.Client.stream requires to write the httpx.HTTPTransport version of the
the httpx.AsyncHTTPTransport declared above.
"""
generator = _stream_generator(method, url, **kwargs)

277
searx/network/client.py
View File

@@ -2,207 +2,114 @@
# pylint: disable=missing-module-docstring, global-statement
import typing as t
from types import TracebackType
import asyncio
import logging
import random
from ssl import SSLContext
import os
import threading
import httpx
from httpx_socks import AsyncProxyTransport
from python_socks import parse_proxy_url, ProxyConnectionError, ProxyTimeoutError, ProxyError
from curl_cffi import AsyncSession, CurlHttpVersion, CurlOpt
from curl_cffi.requests.exceptions import InvalidSchema, RequestException
from searx import logger
from searx.extended_types import SXNG_Response
CertTypes = str | tuple[str, str] | tuple[str, str, str]
SslContextKeyType = tuple[str | None, CertTypes | None, bool, bool]
logger = logger.getChild('searx.network.client')
LOOP: asyncio.AbstractEventLoop = None # pyright: ignore[reportAssignmentType]
SSLCONTEXTS: dict[SslContextKeyType, SSLContext] = {}
# chrome is used by default
DEFAULT_IMPERSONATE = "chrome"
NO_IMPERSONATE = "none"
def shuffle_ciphers(ssl_context: SSLContext):
"""Shuffle httpx's default ciphers of a SSL context randomly.
class AsyncClient(AsyncSession):
""":class:`curl_cffi.AsyncSession` with ``aclose`` / ``is_closed``."""
From `What Is TLS Fingerprint and How to Bypass It`_
def __init__(self, enable_http: bool, **kwargs: t.Any):
self.enable_http = enable_http
self._closed = False
super().__init__(**kwargs)
> When implementing TLS fingerprinting, servers can't operate based on a
> locked-in whitelist database of fingerprints. New fingerprints appear
> when web clients or TLS libraries release new versions. So, they have to
> live off a blocklist database instead.
> ...
> It's safe to leave the first three as is but shuffle the remaining ciphers
> and you can bypass the TLS fingerprint check.
@property
def is_closed(self) -> bool:
return self._closed
.. _What Is TLS Fingerprint and How to Bypass It:
https://www.zenrows.com/blog/what-is-tls-fingerprint#how-to-bypass-tls-fingerprinting
"""
c_list = [cipher["name"] for cipher in ssl_context.get_ciphers()]
sc_list, c_list = c_list[:3], c_list[3:]
random.shuffle(c_list)
ssl_context.set_ciphers(":".join(sc_list + c_list))
def get_sslcontexts(
proxy_url: str | None = None, cert: CertTypes | None = None, verify: bool = True, trust_env: bool = True
) -> SSLContext:
key: SslContextKeyType = (proxy_url, cert, verify, trust_env)
if key not in SSLCONTEXTS:
SSLCONTEXTS[key] = httpx.create_ssl_context(verify, cert, trust_env)
shuffle_ciphers(SSLCONTEXTS[key])
return SSLCONTEXTS[key]
class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport):
"""Block HTTP request
The constructor is blank because httpx.AsyncHTTPTransport.__init__ creates an SSLContext unconditionally:
https://github.com/encode/httpx/blob/0f61aa58d66680c239ce43c8cdd453e7dc532bfc/httpx/_transports/default.py#L271
Each SSLContext consumes more than 500kb of memory, since there is about one network per engine.
In consequence, this class overrides all public methods
For reference: https://github.com/encode/httpx/issues/2298
"""
def __init__(self, *args, **kwargs): # type: ignore
# pylint: disable=super-init-not-called
# this on purpose if the base class is not called
pass
async def handle_async_request(self, request: httpx.Request):
raise httpx.UnsupportedProtocol('HTTP protocol is disabled')
def check_url(self, url: str) -> None:
if not self.enable_http and str(url).startswith("http://"):
raise InvalidSchema("HTTP protocol is disabled")
async def aclose(self) -> None:
pass
async def __aenter__(self):
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None = None,
exc_value: BaseException | None = None,
traceback: TracebackType | None = None,
) -> None:
pass
class AsyncProxyTransportFixed(AsyncProxyTransport):
"""Fix httpx_socks.AsyncProxyTransport
Map python_socks exceptions to httpx.ProxyError exceptions
"""
async def handle_async_request(self, request: httpx.Request):
if self._closed:
return
self._closed = True
try:
return await super().handle_async_request(request)
except ProxyConnectionError as e:
raise httpx.ProxyError("ProxyConnectionError: " + str(e.strerror), request=request) from e
except ProxyTimeoutError as e:
raise httpx.ProxyError("ProxyTimeoutError: " + str(e.args[0]), request=request) from e
except ProxyError as e:
raise httpx.ProxyError("ProxyError: " + str(e.args[0]), request=request) from e
await self.close()
except RequestException:
pass
def get_transport_for_socks_proxy(
verify: bool, http2: bool, local_address: str, proxy_url: str, limit: httpx.Limits, retries: int
):
# support socks5h (requests compatibility):
# https://requests.readthedocs.io/en/master/user/advanced/#socks
# socks5:// hostname is resolved on client side
# socks5h:// hostname is resolved on proxy side
rdns = False
socks5h = 'socks5h://'
if proxy_url.startswith(socks5h):
proxy_url = 'socks5://' + proxy_url[len(socks5h) :]
rdns = True
proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)
_verify = get_sslcontexts(proxy_url, None, verify, True) if verify is True else verify
return AsyncProxyTransportFixed(
proxy_type=proxy_type,
proxy_host=proxy_host,
proxy_port=proxy_port,
username=proxy_username,
password=proxy_password,
rdns=rdns,
verify=_verify, # pyright: ignore[reportArgumentType]
http2=http2,
local_address=local_address,
limits=limit,
retries=retries,
)
def get_transport(
verify: bool, http2: bool, local_address: str, proxy_url: str | None, limit: httpx.Limits, retries: int
):
_verify = get_sslcontexts(None, None, verify, True) if verify is True else verify
return httpx.AsyncHTTPTransport(
# pylint: disable=protected-access
verify=_verify,
http2=http2,
limits=limit,
proxy=httpx._config.Proxy(proxy_url) if proxy_url else None, # pyright: ignore[reportPrivateUsage]
local_address=local_address,
retries=retries,
)
def _proxy_kwargs(proxies: dict[str, str], enable_http: bool) -> dict[str, t.Any]:
"""Map settings.yml proxy keys (``all://``, ``https://``) to curl_cffi."""
mapped: dict[str, str] = {}
all_proxy: str | None = None
for pattern, proxy_url in proxies.items():
if not enable_http and pattern.startswith("http://"):
continue
if pattern.startswith("https"):
mapped["https"] = proxy_url
elif pattern.startswith("http"):
mapped["http"] = proxy_url
else:
all_proxy = proxy_url
if all_proxy:
return {"proxy": all_proxy}
if mapped:
return {"proxies": mapped}
return {}
def new_client(
# pylint: disable=too-many-arguments
enable_http: bool,
verify: bool,
verify: bool | str,
enable_http2: bool,
enable_http3: bool,
max_connections: int,
max_keepalive_connections: int,
keepalive_expiry: float,
proxies: dict[str, str],
local_address: str,
retries: int,
local_address: str | None,
max_redirects: int,
hook_log_response: t.Callable[..., t.Any] | None,
) -> httpx.AsyncClient:
limit = httpx.Limits(
max_connections=max_connections,
max_keepalive_connections=max_keepalive_connections,
keepalive_expiry=keepalive_expiry,
)
# See https://www.python-httpx.org/advanced/#routing
mounts = {}
mounts: None | (dict[str, t.Any | None]) = {}
for pattern, proxy_url in proxies.items():
if not enable_http and pattern.startswith('http://'):
continue
if proxy_url.startswith('socks4://') or proxy_url.startswith('socks5://') or proxy_url.startswith('socks5h://'):
mounts[pattern] = get_transport_for_socks_proxy(
verify, enable_http2, local_address, proxy_url, limit, retries
)
else:
mounts[pattern] = get_transport(verify, enable_http2, local_address, proxy_url, limit, retries)
if not enable_http:
mounts['http://'] = AsyncHTTPTransportNoHttp()
transport = get_transport(verify, enable_http2, local_address, None, limit, retries)
event_hooks = None
if hook_log_response:
event_hooks = {'response': [hook_log_response]}
return httpx.AsyncClient(
transport=transport,
mounts=mounts,
max_redirects=max_redirects,
event_hooks=event_hooks,
)
impersonate: str = DEFAULT_IMPERSONATE,
curl_options: dict[int, t.Any] | None = None,
) -> AsyncClient:
extra_curl = dict(curl_options or {})
cert_file = os.environ.get("SSL_CERT_FILE")
if cert_file:
extra_curl.setdefault(CurlOpt.CAINFO, cert_file)
cert_dir = os.environ.get("SSL_CERT_DIR")
if cert_dir:
extra_curl.setdefault(CurlOpt.CAPATH, cert_dir)
use_impersonate = impersonate not in ("", NO_IMPERSONATE)
kwargs: dict[str, t.Any] = {
"enable_http": enable_http,
"verify": verify,
"max_redirects": max_redirects,
"max_clients": max_connections or 10,
"response_class": SXNG_Response,
"discard_cookies": True,
**_proxy_kwargs(proxies, enable_http),
}
if use_impersonate:
kwargs["impersonate"] = impersonate
kwargs["default_headers"] = True
if local_address:
kwargs["interface"] = local_address
if not enable_http2:
kwargs["http_version"] = CurlHttpVersion.V1_1
elif enable_http3 and not proxies:
kwargs["http_version"] = CurlHttpVersion.V3
else:
kwargs["http_version"] = CurlHttpVersion.V2_0
if extra_curl:
kwargs["curl_options"] = extra_curl
return AsyncClient(**kwargs)
def get_loop() -> asyncio.AbstractEventLoop:
@@ -210,30 +117,18 @@ def get_loop() -> asyncio.AbstractEventLoop:
def init():
# log
for logger_name in (
'httpx',
'httpcore.proxy',
'httpcore.connection',
'httpcore.http11',
'httpcore.http2',
'hpack.hpack',
'hpack.table',
):
logging.getLogger(logger_name).setLevel(logging.WARNING)
logging.getLogger("curl_cffi").setLevel(logging.WARNING)
ready = threading.Event()
# loop
def loop_thread():
global LOOP
LOOP = asyncio.new_event_loop()
ready.set()
LOOP.run_forever()
thread = threading.Thread(
target=loop_thread,
name='asyncio_loop',
daemon=True,
)
thread.start()
threading.Thread(target=loop_thread, name="asyncio_loop", daemon=True).start()
ready.wait()
init()

143
searx/network/network.py
View File

@@ -13,11 +13,16 @@ import asyncio
import ipaddress
from itertools import cycle
import httpx
from curl_cffi import CurlHttpVersion
from curl_cffi.requests.exceptions import (
ConnectionError as CurlConnectionError,
ProxyError,
RequestException,
)
from searx import logger, sxng_debug
from searx.extended_types import SXNG_Response
from .client import new_client, get_loop, AsyncHTTPTransportNoHttp
from .client import DEFAULT_IMPERSONATE, AsyncClient, new_client, get_loop
from .raise_for_httperror import raise_for_httperror
@@ -48,9 +53,8 @@ class Network:
'enable_http',
'verify',
'enable_http2',
'enable_http3',
'max_connections',
'max_keepalive_connections',
'keepalive_expiry',
'local_addresses',
'proxies',
'using_tor_proxy',
@@ -64,6 +68,7 @@ class Network:
)
_TOR_CHECK_RESULT = {}
_CLIENT_KWARGS = ('verify', 'max_redirects', 'impersonate', 'curl_options', 'enable_http3')
def __init__(
# pylint: disable=too-many-arguments
@@ -71,9 +76,8 @@ class Network:
enable_http: bool = True,
verify: bool = True,
enable_http2: bool = False,
enable_http3: bool = False,
max_connections: int = None, # pyright: ignore[reportArgumentType]
max_keepalive_connections: int = None, # pyright: ignore[reportArgumentType]
keepalive_expiry: float = None, # pyright: ignore[reportArgumentType]
proxies: str | dict[str, str] | None = None,
using_tor_proxy: bool = False,
local_addresses: str | list[str] | None = None,
@@ -86,9 +90,8 @@ class Network:
self.enable_http = enable_http
self.verify = verify
self.enable_http2 = enable_http2
self.enable_http3 = enable_http3
self.max_connections = max_connections
self.max_keepalive_connections = max_keepalive_connections
self.keepalive_expiry = keepalive_expiry
self.proxies = proxies
self.using_tor_proxy = using_tor_proxy
self.local_addresses = local_addresses
@@ -137,7 +140,6 @@ class Network:
def iter_proxies(self) -> Generator[tuple[str, list[str]]]:
if not self.proxies:
return
# https://www.python-httpx.org/compatibility/#proxy-keys
if isinstance(self.proxies, str):
yield 'all://', [self.proxies]
else:
@@ -155,62 +157,73 @@ class Network:
# pylint: disable=stop-iteration-return
yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
async def log_response(self, response: httpx.Response):
_HTTP_VERSION = {
int(CurlHttpVersion.V1_0): "HTTP/1.0",
int(CurlHttpVersion.V1_1): "HTTP/1.1",
int(CurlHttpVersion.V2_0): "HTTP/2",
int(CurlHttpVersion.V2TLS): "HTTP/2",
int(CurlHttpVersion.V2_PRIOR_KNOWLEDGE): "HTTP/2",
int(CurlHttpVersion.V3): "HTTP/3",
int(CurlHttpVersion.V3ONLY): "HTTP/3",
}
async def log_response(self, response: SXNG_Response):
request = response.request
status = f"{response.status_code} {response.reason_phrase}"
response_line = f"{response.http_version} {status}"
http_version = self._HTTP_VERSION.get(response.http_version, str(response.http_version))
status = f"{response.status_code} {response.reason}"
response_line = f"{http_version} {status}"
content_type = response.headers.get("Content-Type")
content_type = f' ({content_type})' if content_type else ''
self._logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}')
method = request.method if request else "?"
url = request.url if request else response.url
self._logger.debug(f'HTTP Request: {method} {url} "{response_line}"{content_type}')
@staticmethod
async def check_tor_proxy(client: httpx.AsyncClient, proxies) -> bool:
async def check_tor_proxy(client: AsyncClient, proxies) -> bool:
if proxies in Network._TOR_CHECK_RESULT:
return Network._TOR_CHECK_RESULT[proxies]
result = True
# ignore client._transport because it is not used with all://
for transport in client._mounts.values(): # pylint: disable=protected-access
if isinstance(transport, AsyncHTTPTransportNoHttp):
continue
if getattr(transport, "_pool") and getattr(
# pylint: disable=protected-access
transport._pool, # type: ignore
"_rdns",
False,
):
continue
if not proxies or not all(url.startswith('socks5h://') for _, url in proxies):
Network._TOR_CHECK_RESULT[proxies] = False
return False
response = await client.get("https://check.torproject.org/api/ip", timeout=60)
if not response.json()["IsTor"]:
result = False
result = bool(response.json()["IsTor"])
Network._TOR_CHECK_RESULT[proxies] = result
return result
async def get_client(self, verify: bool | None = None, max_redirects: int | None = None) -> httpx.AsyncClient:
async def get_client(
self,
verify: bool | None = None,
max_redirects: int | None = None,
impersonate: str | None = None,
curl_options: dict[int, t.Any] | None = None,
enable_http3: bool | None = None,
) -> AsyncClient:
verify = self.verify if verify is None else verify
max_redirects = self.max_redirects if max_redirects is None else max_redirects
impersonate = impersonate or DEFAULT_IMPERSONATE
enable_http3 = self.enable_http3 if enable_http3 is None else enable_http3
local_address = next(self._local_addresses_cycle)
proxies = next(self._proxies_cycle) # is a tuple so it can be part of the key
key = (verify, max_redirects, local_address, proxies)
hook_log_response = self.log_response if sxng_debug else None
curl_key = tuple(sorted((int(k), v) for k, v in (curl_options or {}).items()))
key = (verify, max_redirects, local_address, proxies, impersonate, curl_key, enable_http3)
if key not in self._clients or self._clients[key].is_closed:
client = new_client(
self.enable_http,
verify,
self.enable_http2,
enable_http3,
self.max_connections,
self.max_keepalive_connections,
self.keepalive_expiry,
dict(proxies),
local_address,
0,
max_redirects,
hook_log_response,
impersonate=impersonate,
curl_options=curl_options,
)
if self.using_tor_proxy and not await self.check_tor_proxy(client, proxies):
await client.aclose()
raise httpx.ProxyError('Network configuration problem: not using Tor')
raise ProxyError('Network configuration problem: not using Tor')
self._clients[key] = client
return self._clients[key]
@@ -218,22 +231,14 @@ class Network:
async def close_client(client):
try:
await client.aclose()
except httpx.HTTPError:
except RequestException:
pass
await asyncio.gather(*[close_client(client) for client in self._clients.values()], return_exceptions=False)
@staticmethod
def extract_kwargs_clients(kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
kwargs_clients: dict[str, t.Any] = {}
if 'verify' in kwargs:
kwargs_clients['verify'] = kwargs.pop('verify')
if 'max_redirects' in kwargs:
kwargs_clients['max_redirects'] = kwargs.pop('max_redirects')
if 'allow_redirects' in kwargs:
# see https://github.com/encode/httpx/pull/1808
kwargs['follow_redirects'] = kwargs.pop('allow_redirects')
return kwargs_clients
return {key: kwargs.pop(key) for key in Network._CLIENT_KWARGS if key in kwargs}
@staticmethod
def extract_do_raise_for_httperror(kwargs: dict[str, t.Any]):
@@ -243,23 +248,18 @@ class Network:
del kwargs['raise_for_httperror']
return do_raise_for_httperror
def patch_response(self, response: httpx.Response, do_raise_for_httperror: bool) -> SXNG_Response:
if isinstance(response, httpx.Response):
response = t.cast(SXNG_Response, response)
# requests compatibility (response is not streamed)
# see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses
response.ok = not response.is_error
# raise an exception
if do_raise_for_httperror:
try:
raise_for_httperror(response)
except:
self._logger.warning(f"HTTP Request failed: {response.request.method} {response.request.url}")
raise
def patch_response(self, response: SXNG_Response, do_raise_for_httperror: bool) -> SXNG_Response:
if do_raise_for_httperror:
try:
raise_for_httperror(response)
except:
method = response.request.method if response.request else "?"
url = response.request.url if response.request else response.url
self._logger.warning(f"HTTP Request failed: {method} {url}")
raise
return response
def is_valid_response(self, response: httpx.Response):
def is_valid_response(self, response: SXNG_Response):
# pylint: disable=too-many-boolean-expressions
if (
(self.retry_on_http_error is True and 400 <= response.status_code <= 599)
@@ -276,26 +276,29 @@ class Network:
kwargs_clients = Network.extract_kwargs_clients(kwargs)
while retries >= 0: # pragma: no cover
client = await self.get_client(**kwargs_clients)
cookies = kwargs.pop("cookies", None)
client.cookies = httpx.Cookies(cookies)
try:
method = method.upper()
client.check_url(url)
if stream:
return client.stream(method, url, **kwargs)
response = await client.request(method, url, **kwargs)
if sxng_debug:
await self.log_response(response)
if self.is_valid_response(response) or retries <= 0:
return self.patch_response(response, do_raise_for_httperror)
except httpx.RemoteProtocolError as e:
await client.aclose()
except CurlConnectionError as e:
if not was_disconnected:
# the server has closed the connection:
# try again without decreasing the retries variable & with a new HTTP client
was_disconnected = True
await client.aclose()
self._logger.warning('httpx.RemoteProtocolError: the server has disconnected, retrying')
self._logger.warning('ConnectionError: the server has disconnected, retrying')
continue
if retries <= 0:
raise e
except (httpx.RequestError, httpx.HTTPStatusError) as e:
except RequestException as e:
if retries <= 0:
raise e
retries -= 1
@@ -346,15 +349,12 @@ def initialize(
settings_engines = settings_engines or settings['engines']
settings_outgoing = settings_outgoing or settings['outgoing']
# default parameters for AsyncHTTPTransport
# see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121 # pylint: disable=line-too-long
default_params: dict[str, t.Any] = {
'enable_http': False,
'verify': settings_outgoing['verify'],
'enable_http2': settings_outgoing['enable_http2'],
'enable_http3': False,
'max_connections': settings_outgoing['pool_connections'],
'max_keepalive_connections': settings_outgoing['pool_maxsize'],
'keepalive_expiry': settings_outgoing['keepalive_expiry'],
'local_addresses': settings_outgoing['source_ips'],
'using_tor_proxy': settings_outgoing['using_tor_proxy'],
'proxies': settings_outgoing['proxies'],
@@ -424,9 +424,6 @@ def initialize(
def done():
"""Close all HTTP client
Avoid a warning at exit
See https://github.com/encode/httpx/pull/2026
Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
So Network.aclose is called here using atexit.register
"""

7
searx/network/raise_for_httperror.py
View File

@@ -59,13 +59,10 @@ def raise_for_captcha(resp: "SXNG_Response"):
def raise_for_httperror(resp: "SXNG_Response") -> None:
"""Raise exception for an HTTP response is an error.
Args:
resp (requests.Response): Response to check
"""Raise an exception if the HTTP response is an error.
Raises:
requests.HTTPError: raise by resp.raise_for_status()
curl_cffi.requests.exceptions.HTTPError: raised by resp.raise_for_status()
searx.exceptions.SearxEngineAccessDeniedException: raise when the HTTP status code is 402 or 403.
searx.exceptions.SearxEngineTooManyRequestsException: raise when the HTTP status code is 429.
searx.exceptions.SearxEngineCaptchaException: raise when if CATPCHA challenge is detected.