mirror of
https://github.com/searxng/searxng.git
synced 2026-09-11 16:56:05 +00:00
135 lines
3.8 KiB
Python
135 lines
3.8 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# pylint: disable=missing-module-docstring, global-statement
|
|
|
|
import typing as t
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import threading
|
|
|
|
from curl_cffi import AsyncSession, CurlHttpVersion, CurlOpt
|
|
from curl_cffi.requests.exceptions import InvalidSchema, RequestException
|
|
|
|
from searx.extended_types import SXNG_Response
|
|
|
|
LOOP: asyncio.AbstractEventLoop = None # pyright: ignore[reportAssignmentType]
|
|
|
|
# chrome is used by default
|
|
DEFAULT_IMPERSONATE = "chrome"
|
|
NO_IMPERSONATE = "none"
|
|
|
|
|
|
class AsyncClient(AsyncSession):
|
|
""":class:`curl_cffi.AsyncSession` with ``aclose`` / ``is_closed``."""
|
|
|
|
def __init__(self, enable_http: bool, **kwargs: t.Any):
|
|
self.enable_http = enable_http
|
|
self._closed = False
|
|
super().__init__(**kwargs)
|
|
|
|
@property
|
|
def is_closed(self) -> bool:
|
|
return self._closed
|
|
|
|
def check_url(self, url: str) -> None:
|
|
if not self.enable_http and str(url).startswith("http://"):
|
|
raise InvalidSchema("HTTP protocol is disabled")
|
|
|
|
async def aclose(self) -> None:
|
|
if self._closed:
|
|
return
|
|
self._closed = True
|
|
try:
|
|
await self.close()
|
|
except RequestException:
|
|
pass
|
|
|
|
|
|
def _proxy_kwargs(proxies: dict[str, str], enable_http: bool) -> dict[str, t.Any]:
|
|
"""Map settings.yml proxy keys (``all://``, ``https://``) to curl_cffi."""
|
|
mapped: dict[str, str] = {}
|
|
all_proxy: str | None = None
|
|
for pattern, proxy_url in proxies.items():
|
|
if not enable_http and pattern.startswith("http://"):
|
|
continue
|
|
if pattern.startswith("https"):
|
|
mapped["https"] = proxy_url
|
|
elif pattern.startswith("http"):
|
|
mapped["http"] = proxy_url
|
|
else:
|
|
all_proxy = proxy_url
|
|
if all_proxy:
|
|
return {"proxy": all_proxy}
|
|
if mapped:
|
|
return {"proxies": mapped}
|
|
return {}
|
|
|
|
|
|
def new_client(
|
|
# pylint: disable=too-many-arguments
|
|
enable_http: bool,
|
|
verify: bool | str,
|
|
enable_http2: bool,
|
|
enable_http3: bool,
|
|
max_connections: int,
|
|
proxies: dict[str, str],
|
|
local_address: str | None,
|
|
max_redirects: int,
|
|
impersonate: str = DEFAULT_IMPERSONATE,
|
|
curl_options: dict[int, t.Any] | None = None,
|
|
) -> AsyncClient:
|
|
extra_curl = dict(curl_options or {})
|
|
cert_file = os.environ.get("SSL_CERT_FILE")
|
|
if cert_file:
|
|
extra_curl.setdefault(CurlOpt.CAINFO, cert_file)
|
|
cert_dir = os.environ.get("SSL_CERT_DIR")
|
|
if cert_dir:
|
|
extra_curl.setdefault(CurlOpt.CAPATH, cert_dir)
|
|
use_impersonate = impersonate not in ("", NO_IMPERSONATE)
|
|
kwargs: dict[str, t.Any] = {
|
|
"enable_http": enable_http,
|
|
"verify": verify,
|
|
"max_redirects": max_redirects,
|
|
"max_clients": max_connections or 10,
|
|
"response_class": SXNG_Response,
|
|
"discard_cookies": True,
|
|
**_proxy_kwargs(proxies, enable_http),
|
|
}
|
|
if use_impersonate:
|
|
kwargs["impersonate"] = impersonate
|
|
kwargs["default_headers"] = True
|
|
if local_address:
|
|
kwargs["interface"] = local_address
|
|
if not enable_http2:
|
|
kwargs["http_version"] = CurlHttpVersion.V1_1
|
|
elif enable_http3 and not proxies:
|
|
kwargs["http_version"] = CurlHttpVersion.V3
|
|
else:
|
|
kwargs["http_version"] = CurlHttpVersion.V2_0
|
|
if extra_curl:
|
|
kwargs["curl_options"] = extra_curl
|
|
return AsyncClient(**kwargs)
|
|
|
|
|
|
def get_loop() -> asyncio.AbstractEventLoop:
|
|
return LOOP
|
|
|
|
|
|
def init():
|
|
logging.getLogger("curl_cffi").setLevel(logging.WARNING)
|
|
|
|
ready = threading.Event()
|
|
|
|
def loop_thread():
|
|
global LOOP
|
|
LOOP = asyncio.new_event_loop()
|
|
ready.set()
|
|
LOOP.run_forever()
|
|
|
|
threading.Thread(target=loop_thread, name="asyncio_loop", daemon=True).start()
|
|
ready.wait()
|
|
|
|
|
|
init()
|