mirror of
https://github.com/searxng/searxng.git
synced 2026-09-12 01:06:04 +00:00
[mod] network: migrate to curl_cffi
This commit is contained in:
vojkovic
Brock Vojkovic
@@ -2,207 +2,114 @@
|
||||
# pylint: disable=missing-module-docstring, global-statement
|
||||
|
||||
import typing as t
|
||||
from types import TracebackType
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from ssl import SSLContext
|
||||
import os
|
||||
import threading
|
||||
|
||||
import httpx
|
||||
from httpx_socks import AsyncProxyTransport
|
||||
from python_socks import parse_proxy_url, ProxyConnectionError, ProxyTimeoutError, ProxyError
|
||||
from curl_cffi import AsyncSession, CurlHttpVersion, CurlOpt
|
||||
from curl_cffi.requests.exceptions import InvalidSchema, RequestException
|
||||
|
||||
from searx import logger
|
||||
from searx.extended_types import SXNG_Response
|
||||
|
||||
CertTypes = str | tuple[str, str] | tuple[str, str, str]
|
||||
SslContextKeyType = tuple[str | None, CertTypes | None, bool, bool]
|
||||
|
||||
logger = logger.getChild('searx.network.client')
|
||||
LOOP: asyncio.AbstractEventLoop = None # pyright: ignore[reportAssignmentType]
|
||||
|
||||
SSLCONTEXTS: dict[SslContextKeyType, SSLContext] = {}
|
||||
# chrome is used by default
|
||||
DEFAULT_IMPERSONATE = "chrome"
|
||||
NO_IMPERSONATE = "none"
|
||||
|
||||
|
||||
def shuffle_ciphers(ssl_context: SSLContext):
|
||||
"""Shuffle httpx's default ciphers of a SSL context randomly.
|
||||
class AsyncClient(AsyncSession):
|
||||
""":class:`curl_cffi.AsyncSession` with ``aclose`` / ``is_closed``."""
|
||||
|
||||
From `What Is TLS Fingerprint and How to Bypass It`_
|
||||
def __init__(self, enable_http: bool, **kwargs: t.Any):
|
||||
self.enable_http = enable_http
|
||||
self._closed = False
|
||||
super().__init__(**kwargs)
|
||||
|
||||
> When implementing TLS fingerprinting, servers can't operate based on a
|
||||
> locked-in whitelist database of fingerprints. New fingerprints appear
|
||||
> when web clients or TLS libraries release new versions. So, they have to
|
||||
> live off a blocklist database instead.
|
||||
> ...
|
||||
> It's safe to leave the first three as is but shuffle the remaining ciphers
|
||||
> and you can bypass the TLS fingerprint check.
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
return self._closed
|
||||
|
||||
.. _What Is TLS Fingerprint and How to Bypass It:
|
||||
https://www.zenrows.com/blog/what-is-tls-fingerprint#how-to-bypass-tls-fingerprinting
|
||||
|
||||
"""
|
||||
c_list = [cipher["name"] for cipher in ssl_context.get_ciphers()]
|
||||
sc_list, c_list = c_list[:3], c_list[3:]
|
||||
random.shuffle(c_list)
|
||||
ssl_context.set_ciphers(":".join(sc_list + c_list))
|
||||
|
||||
|
||||
def get_sslcontexts(
|
||||
proxy_url: str | None = None, cert: CertTypes | None = None, verify: bool = True, trust_env: bool = True
|
||||
) -> SSLContext:
|
||||
key: SslContextKeyType = (proxy_url, cert, verify, trust_env)
|
||||
if key not in SSLCONTEXTS:
|
||||
SSLCONTEXTS[key] = httpx.create_ssl_context(verify, cert, trust_env)
|
||||
shuffle_ciphers(SSLCONTEXTS[key])
|
||||
return SSLCONTEXTS[key]
|
||||
|
||||
|
||||
class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport):
|
||||
"""Block HTTP request
|
||||
|
||||
The constructor is blank because httpx.AsyncHTTPTransport.__init__ creates an SSLContext unconditionally:
|
||||
https://github.com/encode/httpx/blob/0f61aa58d66680c239ce43c8cdd453e7dc532bfc/httpx/_transports/default.py#L271
|
||||
|
||||
Each SSLContext consumes more than 500kb of memory, since there is about one network per engine.
|
||||
|
||||
In consequence, this class overrides all public methods
|
||||
|
||||
For reference: https://github.com/encode/httpx/issues/2298
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs): # type: ignore
|
||||
# pylint: disable=super-init-not-called
|
||||
# this on purpose if the base class is not called
|
||||
pass
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request):
|
||||
raise httpx.UnsupportedProtocol('HTTP protocol is disabled')
|
||||
def check_url(self, url: str) -> None:
|
||||
if not self.enable_http and str(url).startswith("http://"):
|
||||
raise InvalidSchema("HTTP protocol is disabled")
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None = None,
|
||||
exc_value: BaseException | None = None,
|
||||
traceback: TracebackType | None = None,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class AsyncProxyTransportFixed(AsyncProxyTransport):
|
||||
"""Fix httpx_socks.AsyncProxyTransport
|
||||
|
||||
Map python_socks exceptions to httpx.ProxyError exceptions
|
||||
"""
|
||||
|
||||
async def handle_async_request(self, request: httpx.Request):
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
try:
|
||||
return await super().handle_async_request(request)
|
||||
except ProxyConnectionError as e:
|
||||
raise httpx.ProxyError("ProxyConnectionError: " + str(e.strerror), request=request) from e
|
||||
except ProxyTimeoutError as e:
|
||||
raise httpx.ProxyError("ProxyTimeoutError: " + str(e.args[0]), request=request) from e
|
||||
except ProxyError as e:
|
||||
raise httpx.ProxyError("ProxyError: " + str(e.args[0]), request=request) from e
|
||||
await self.close()
|
||||
except RequestException:
|
||||
pass
|
||||
|
||||
|
||||
def get_transport_for_socks_proxy(
|
||||
verify: bool, http2: bool, local_address: str, proxy_url: str, limit: httpx.Limits, retries: int
|
||||
):
|
||||
# support socks5h (requests compatibility):
|
||||
# https://requests.readthedocs.io/en/master/user/advanced/#socks
|
||||
# socks5:// hostname is resolved on client side
|
||||
# socks5h:// hostname is resolved on proxy side
|
||||
rdns = False
|
||||
socks5h = 'socks5h://'
|
||||
if proxy_url.startswith(socks5h):
|
||||
proxy_url = 'socks5://' + proxy_url[len(socks5h) :]
|
||||
rdns = True
|
||||
|
||||
proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)
|
||||
_verify = get_sslcontexts(proxy_url, None, verify, True) if verify is True else verify
|
||||
return AsyncProxyTransportFixed(
|
||||
proxy_type=proxy_type,
|
||||
proxy_host=proxy_host,
|
||||
proxy_port=proxy_port,
|
||||
username=proxy_username,
|
||||
password=proxy_password,
|
||||
rdns=rdns,
|
||||
verify=_verify, # pyright: ignore[reportArgumentType]
|
||||
http2=http2,
|
||||
local_address=local_address,
|
||||
limits=limit,
|
||||
retries=retries,
|
||||
)
|
||||
|
||||
|
||||
def get_transport(
|
||||
verify: bool, http2: bool, local_address: str, proxy_url: str | None, limit: httpx.Limits, retries: int
|
||||
):
|
||||
_verify = get_sslcontexts(None, None, verify, True) if verify is True else verify
|
||||
return httpx.AsyncHTTPTransport(
|
||||
# pylint: disable=protected-access
|
||||
verify=_verify,
|
||||
http2=http2,
|
||||
limits=limit,
|
||||
proxy=httpx._config.Proxy(proxy_url) if proxy_url else None, # pyright: ignore[reportPrivateUsage]
|
||||
local_address=local_address,
|
||||
retries=retries,
|
||||
)
|
||||
def _proxy_kwargs(proxies: dict[str, str], enable_http: bool) -> dict[str, t.Any]:
|
||||
"""Map settings.yml proxy keys (``all://``, ``https://``) to curl_cffi."""
|
||||
mapped: dict[str, str] = {}
|
||||
all_proxy: str | None = None
|
||||
for pattern, proxy_url in proxies.items():
|
||||
if not enable_http and pattern.startswith("http://"):
|
||||
continue
|
||||
if pattern.startswith("https"):
|
||||
mapped["https"] = proxy_url
|
||||
elif pattern.startswith("http"):
|
||||
mapped["http"] = proxy_url
|
||||
else:
|
||||
all_proxy = proxy_url
|
||||
if all_proxy:
|
||||
return {"proxy": all_proxy}
|
||||
if mapped:
|
||||
return {"proxies": mapped}
|
||||
return {}
|
||||
|
||||
|
||||
def new_client(
|
||||
# pylint: disable=too-many-arguments
|
||||
enable_http: bool,
|
||||
verify: bool,
|
||||
verify: bool | str,
|
||||
enable_http2: bool,
|
||||
enable_http3: bool,
|
||||
max_connections: int,
|
||||
max_keepalive_connections: int,
|
||||
keepalive_expiry: float,
|
||||
proxies: dict[str, str],
|
||||
local_address: str,
|
||||
retries: int,
|
||||
local_address: str | None,
|
||||
max_redirects: int,
|
||||
hook_log_response: t.Callable[..., t.Any] | None,
|
||||
) -> httpx.AsyncClient:
|
||||
limit = httpx.Limits(
|
||||
max_connections=max_connections,
|
||||
max_keepalive_connections=max_keepalive_connections,
|
||||
keepalive_expiry=keepalive_expiry,
|
||||
)
|
||||
# See https://www.python-httpx.org/advanced/#routing
|
||||
mounts = {}
|
||||
mounts: None | (dict[str, t.Any | None]) = {}
|
||||
for pattern, proxy_url in proxies.items():
|
||||
if not enable_http and pattern.startswith('http://'):
|
||||
continue
|
||||
if proxy_url.startswith('socks4://') or proxy_url.startswith('socks5://') or proxy_url.startswith('socks5h://'):
|
||||
mounts[pattern] = get_transport_for_socks_proxy(
|
||||
verify, enable_http2, local_address, proxy_url, limit, retries
|
||||
)
|
||||
else:
|
||||
mounts[pattern] = get_transport(verify, enable_http2, local_address, proxy_url, limit, retries)
|
||||
|
||||
if not enable_http:
|
||||
mounts['http://'] = AsyncHTTPTransportNoHttp()
|
||||
|
||||
transport = get_transport(verify, enable_http2, local_address, None, limit, retries)
|
||||
|
||||
event_hooks = None
|
||||
if hook_log_response:
|
||||
event_hooks = {'response': [hook_log_response]}
|
||||
|
||||
return httpx.AsyncClient(
|
||||
transport=transport,
|
||||
mounts=mounts,
|
||||
max_redirects=max_redirects,
|
||||
event_hooks=event_hooks,
|
||||
)
|
||||
impersonate: str = DEFAULT_IMPERSONATE,
|
||||
curl_options: dict[int, t.Any] | None = None,
|
||||
) -> AsyncClient:
|
||||
extra_curl = dict(curl_options or {})
|
||||
cert_file = os.environ.get("SSL_CERT_FILE")
|
||||
if cert_file:
|
||||
extra_curl.setdefault(CurlOpt.CAINFO, cert_file)
|
||||
cert_dir = os.environ.get("SSL_CERT_DIR")
|
||||
if cert_dir:
|
||||
extra_curl.setdefault(CurlOpt.CAPATH, cert_dir)
|
||||
use_impersonate = impersonate not in ("", NO_IMPERSONATE)
|
||||
kwargs: dict[str, t.Any] = {
|
||||
"enable_http": enable_http,
|
||||
"verify": verify,
|
||||
"max_redirects": max_redirects,
|
||||
"max_clients": max_connections or 10,
|
||||
"response_class": SXNG_Response,
|
||||
"discard_cookies": True,
|
||||
**_proxy_kwargs(proxies, enable_http),
|
||||
}
|
||||
if use_impersonate:
|
||||
kwargs["impersonate"] = impersonate
|
||||
kwargs["default_headers"] = True
|
||||
if local_address:
|
||||
kwargs["interface"] = local_address
|
||||
if not enable_http2:
|
||||
kwargs["http_version"] = CurlHttpVersion.V1_1
|
||||
elif enable_http3 and not proxies:
|
||||
kwargs["http_version"] = CurlHttpVersion.V3
|
||||
else:
|
||||
kwargs["http_version"] = CurlHttpVersion.V2_0
|
||||
if extra_curl:
|
||||
kwargs["curl_options"] = extra_curl
|
||||
return AsyncClient(**kwargs)
|
||||
|
||||
|
||||
def get_loop() -> asyncio.AbstractEventLoop:
|
||||
@@ -210,30 +117,18 @@ def get_loop() -> asyncio.AbstractEventLoop:
|
||||
|
||||
|
||||
def init():
|
||||
# log
|
||||
for logger_name in (
|
||||
'httpx',
|
||||
'httpcore.proxy',
|
||||
'httpcore.connection',
|
||||
'httpcore.http11',
|
||||
'httpcore.http2',
|
||||
'hpack.hpack',
|
||||
'hpack.table',
|
||||
):
|
||||
logging.getLogger(logger_name).setLevel(logging.WARNING)
|
||||
logging.getLogger("curl_cffi").setLevel(logging.WARNING)
|
||||
|
||||
ready = threading.Event()
|
||||
|
||||
# loop
|
||||
def loop_thread():
|
||||
global LOOP
|
||||
LOOP = asyncio.new_event_loop()
|
||||
ready.set()
|
||||
LOOP.run_forever()
|
||||
|
||||
thread = threading.Thread(
|
||||
target=loop_thread,
|
||||
name='asyncio_loop',
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
threading.Thread(target=loop_thread, name="asyncio_loop", daemon=True).start()
|
||||
ready.wait()
|
||||
|
||||
|
||||
init()
|
||||
|
||||
Reference in New Issue
Block a user