2 Commits

Author SHA1 Message Date
Ivan Gabaldon
00eaf1157d fmt 2026-08-21 22:53:43 +02:00
dependabot[bot]
f92003b360 [upd] pypi: Bump black from 25.9.0 to 26.5.1
Bumps [black](https://github.com/psf/black) from 25.9.0 to 26.5.1.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/compare/25.9.0...26.5.1)

---
updated-dependencies:
- dependency-name: black
  dependency-version: 26.5.1
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-21 07:24:43 +00:00
221 changed files with 8286 additions and 30780 deletions

View File

@@ -62,7 +62,7 @@ jobs:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Setup QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -105,9 +105,8 @@ jobs:
arch: amd64
- runner: ubuntu-26.04-arm
arch: arm64
# FIXME: https://github.com/searxng/searxng/pull/6655#issuecomment-5550293085
# - runner: ubuntu-26.04-arm
# arch: armv7
- runner: ubuntu-26.04-arm
arch: armv7
steps:
- name: Login to GHCR
@@ -118,7 +117,7 @@ jobs:
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Setup QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

View File

@@ -31,7 +31,7 @@ jobs:
- update_external_bangs.py
- update_firefox_version.py
- update_engine_traits.py
- update_wikidata.py
- update_wikidata_units.py
- update_engine_descriptions.py
permissions:

View File

@@ -2,7 +2,7 @@
/*
this file is generated automatically by searxng_extra/update/update_pygments.py
using pygments version 2.21.0:
using pygments version 2.20.0:
./manage templates.simple.pygments
*/
@@ -114,14 +114,14 @@
.gd { color: #FF4689 } /* Generic.Deleted */
.ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
.ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.gr { color: #FF4689 } /* Generic.Error */
.gr { color: #F8F8F2 } /* Generic.Error */
.gh { color: #F8F8F2 } /* Generic.Heading */
.gi { color: #A6E22E } /* Generic.Inserted */
.go { color: #66D9EF } /* Generic.Output */
.gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
.gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
.gu { color: #959077 } /* Generic.Subheading */
.gt { color: #66D9EF } /* Generic.Traceback */
.gt { color: #F8F8F2 } /* Generic.Traceback */
.kc { color: #66D9EF } /* Keyword.Constant */
.kd { color: #66D9EF } /* Keyword.Declaration */
.kn { color: #FF4689 } /* Keyword.Namespace */
@@ -132,7 +132,7 @@
.m { color: #AE81FF } /* Literal.Number */
.s { color: #E6DB74 } /* Literal.String */
.na { color: #A6E22E } /* Name.Attribute */
.nb { color: #A6E22E } /* Name.Builtin */
.nb { color: #F8F8F2 } /* Name.Builtin */
.nc { color: #A6E22E } /* Name.Class */
.no { color: #66D9EF } /* Name.Constant */
.nd { color: #A6E22E } /* Name.Decorator */
@@ -166,7 +166,7 @@
.sr { color: #E6DB74 } /* Literal.String.Regex */
.s1 { color: #E6DB74 } /* Literal.String.Single */
.ss { color: #E6DB74 } /* Literal.String.Symbol */
.bp { color: #A6E22E } /* Name.Builtin.Pseudo */
.bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
.fm { color: #A6E22E } /* Name.Function.Magic */
.vc { color: #F8F8F2 } /* Name.Variable.Class */
.vg { color: #F8F8F2 } /* Name.Variable.Global */

File diff suppressed because it is too large Load Diff

View File

@@ -29,21 +29,21 @@
"swiped-events": "1.2.0"
},
"devDependencies": {
"@biomejs/biome": "2.5.12",
"@types/node": "^26.5.0",
"@biomejs/biome": "2.5.7",
"@types/node": "^26.2.0",
"browserslist": "^4.28.8",
"browserslist-to-esbuild": "^2.1.1",
"edge.js": "^6.5.1",
"less": "^4.9.0",
"less": "^4.8.1",
"mathjs": "^15.2.0",
"sharp": "~0.35.4",
"sharp": "~0.35.3",
"sort-package-json": "^4.0.0",
"stylelint": "^17.14.1",
"stylelint-config-standard-less": "^4.1.0",
"stylelint-prettier": "^5.0.3",
"svgo": "^4.1.0",
"svgo": "^4.0.2",
"typescript": "~7.0.2",
"vite": "^8.2.2",
"vite": "^8.2.1",
"vite-bundle-analyzer": "^1.3.9"
}
}

View File

@@ -58,9 +58,10 @@ engine is shown. Most of the options have a default value or even are optional.
# overwrite values from section 'outgoing:'
enable_http2: false
enable_http3: false
retries: 1
max_connections: 100
max_keepalive_connections: 10
keepalive_expiry: 5.0
using_tor_proxy: false
proxies:
http:
@@ -162,16 +163,6 @@ engine is shown. Most of the options have a default value or even are optional.
``enable_http`` : optional
Enable HTTP for this engine (by default only HTTPS is enabled).
``enable_http3`` : optional
Use HTTP/3 (falls back to HTTP/2). Default ``false``.
Ignored when a proxy is set.
.. hint::
HTTP/3 places demands on the IP infrastructure that are not met in every
environment. Enable this option only if you are aware of these requirements
and the extent to which they are met.
``retry_on_http_error`` : optional
Retry request on some HTTP status code.
@@ -188,12 +179,20 @@ engine is shown. Most of the options have a default value or even are optional.
Using tor proxy (``true``) or not (``false``) for this engine. The default is
taken from ``using_tor_proxy`` of the :ref:`settings outgoing`.
.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
``max_keepalive_connection#s`` :
`Pool limit configuration`_, overwrites value ``pool_maxsize`` from
:ref:`settings outgoing` for this engine.
``max_connections`` :
`Pool limit configuration`_, overwrites value ``pool_connections`` from
:ref:`settings outgoing` for this engine.
``keepalive_expiry`` :
`Pool limit configuration`_, overwrites value ``keepalive_expiry`` from
:ref:`settings outgoing` for this engine.
.. _private engines:

View File

@@ -12,12 +12,20 @@ Communication with search engines.
request_timeout: 2.0 # default timeout in seconds, can be override by engine
max_request_timeout: 10.0 # the maximum timeout in seconds
useragent_suffix: "" # information like an email address to the administrator
pool_connections: 100 # Maximum number of concurrent connections (default: 100)
enable_http2: true # Enables the use of HTTP2
pool_connections: 100 # Maximum number of allowable connections, or null
# for no limits. The default is 100.
pool_maxsize: 10 # Number of allowable keep-alive connections, or null
# to always allow. The default is 10.
enable_http2: true # See https://www.python-httpx.org/http2/
# uncomment below section if you want to use a custom server certificate
# see https://www.python-httpx.org/advanced/#changing-the-verification-defaults
# and https://www.python-httpx.org/compatibility/#ssl-configuration
# verify: ~/.mitmproxy/mitmproxy-ca-cert.cer
#
# uncomment below section if you want to use a proxy
# uncomment below section if you want to use a proxyq see: SOCKS proxies
# https://2.python-requests.org/en/latest/user/advanced/#proxies
# are also supported: see
# https://2.python-requests.org/en/latest/user/advanced/#socks
#
# proxies:
# all://:
@@ -38,26 +46,30 @@ Communication with search engines.
timeout to load). Can be override by ``timeout`` in the :ref:`settings engines`.
``useragent_suffix`` :
Suffix to add when an engine's User-Agent is set via searxng_useragent().
Contact info here may be useful to avoid an engine blocking you.
Suffix to the user-agent SearXNG uses to send requests to others engines. If an
engine wish to block you, a contact info here may be useful to avoid that.
.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
``pool_maxsize``:
Number of allowable keep-alive connections, or ``null`` to always allow. The
default is 10. See ``max_keepalive_connections`` `Pool limit configuration`_.
``pool_connections`` :
Maximum number of concurrent connections. The default is 100.
See ``max_clients`` `Pool limit configuration`_.
Maximum number of allowable connections, or ``null`` # for no limits. The
default is 100. See ``max_connections`` `Pool limit configuration`_.
.. _curl_cffi proxies: https://curl-cffi.readthedocs.io/en/latest/quick_start.html
``keepalive_expiry`` :
Number of seconds to keep a connection in the pool. By default 5.0 seconds.
See ``keepalive_expiry`` `Pool limit configuration`_.
.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying
``proxies`` :
Define one or more proxies you wish to use, see `curl_cffi proxies`_.
Define one or more proxies you wish to use, see `httpx proxies`_.
If there are more than one proxy for one protocol (http, https),
requests to the engines are distributed in a round-robin fashion.
HTTP, HTTPS, SOCKS4, SOCKS5 and SOCKS5h proxies are supported
(``http://``, ``https://``, ``socks4://``, ``socks5://``, ``socks5h://``). You should
use ``socks5h://`` when using Tor so hostnames are resolved by the proxy.
``source_ips`` :
If you use multiple network interfaces, define from which IP the requests must
be made. Example:
@@ -75,15 +87,18 @@ Communication with search engines.
different proxy and source ip.
``enable_http2`` :
Enable by default (HTTP/2). Set to ``false`` to force HTTP/1.1.
HTTP/3 is opt-in per engine (``enable_http3``).
Enable by default. Set to ``false`` to disable HTTP/2.
.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults
.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration
``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR``
HTTPS verification uses the OS's trust store by default.
Set a path to use a custom CA file.
Allow to specify a path to certificate.
see `httpx verification defaults`_.
In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and
``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.
see `httpx ssl configuration`_.
``max_redirects`` :
30 by default. Maximum redirect before it is an error.

View File

@@ -143,7 +143,7 @@ parameters with default value can be redefined for special purposes.
data dict ``{}``
cookies dict ``{}``
verify bool ``True``
headers.User-Agent str ``''``
headers.User-Agent str a random User-Agent
category str current category, like ``'general'``
safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict)
time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year``
@@ -229,8 +229,6 @@ following parameters can be used to specify a search request:
max_redirects int maximum redirects, hard limit
soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine
raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300
impersonate str curl_cffi impersonate target (default: chrome, none to disable)
curl_options dict Any extra libcurl options for the request
=================== =========== ==========================================================================

View File

@@ -0,0 +1,8 @@
.. _cara engine:
===========
Cara Images
===========
.. automodule:: searx.engines.cara
:members:

View File

@@ -1,8 +0,0 @@
.. _europepmc engine:
==========
Europe PMC
==========
.. automodule:: searx.engines.europepmc
:members:

View File

@@ -80,8 +80,8 @@ same environment, here are a few examples::
# to test one of the update scripts
(dev.env)$ searxng_extra/update/update_engine_traits.py --help
# to test the update of the wikidata units and property names
(dev.env)$ searxng_extra/update/update_wikidata.py
# to test the update of the wikidata units
(dev.env)$ searxng_extra/update/update_wikidata_units.py
.. sidebar:: further read

View File

@@ -286,7 +286,7 @@ content becomes smart.
files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst```
pull request :pull:`4` ``:pull:`4```
patch :patch:`af2cae6` ``:patch:`af2cae6```
PyPi package :pypi:`curl_cffi` ``:pypi:`curl_cffi```
PyPi package :pypi:`httpx` ``:pypi:`httpx```
manual page man :man:`bash` ``:man:`bash```
intersphinx_
--------------------------------------------------------------------------------------------------

View File

@@ -90,10 +90,10 @@ Scripts to update static data in :origin:`searx/data/`
:members:
``update_wikidata.py``
``update_wikidata_units.py``
============================
:origin:`[source] <searxng_extra/update/update_wikidata.py>`
:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
.. automodule:: searxng_extra.update.update_wikidata
.. automodule:: searxng_extra.update.update_wikidata_units
:members:

View File

@@ -1,10 +1,10 @@
mock==5.2.0
nose2[coverage_plugin]==0.16.0
cov-core==1.15.0
black==25.9.0
pylint==4.0.8
black==26.5.1
pylint==4.0.7
splinter==0.21.0
selenium==4.48.0
selenium==4.47.0
Sphinx==8.2.3;python_version <= "3.11"
Sphinx==9.1.0; python_version > "3.11"
sphinx-issues==6.0.0
@@ -23,6 +23,6 @@ coloredlogs==15.0.1
docutils>=0.21.2;python_version <= "3.11"
docutils>=0.22.4; python_version > "3.11"
parameterized==0.9.0
granian[reload]==2.8.2
basedpyright==1.39.10
granian[reload]==2.8.1
basedpyright==1.39.9
types-lxml==2026.2.16

View File

@@ -1,2 +1,2 @@
granian==2.8.2
granian[pname]==2.8.2
granian==2.8.1
granian[pname]==2.8.1

View File

@@ -3,15 +3,17 @@ babel==2.18.0
flask-babel==4.0.0
flask==3.1.3
jinja2==3.1.6
lxml==6.1.2
pygments==2.21.0
lxml==6.1.1
pygments==2.20.0
python-dateutil==2.9.0.post0
pyyaml==6.0.3
curl_cffi==0.16.1
httpx[http2]==0.28.1
httpx-socks[asyncio]==0.13.1
sniffio==1.3.1
valkey==6.1.1
markdown-it-py==4.2.0
msgspec==0.21.1
typer==0.27.2
typer==0.27.1
isodate==0.7.2
whitenoise==6.12.0
typing-extensions==4.16.0

View File

@@ -1,5 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Implementation of the :py:obj:`preference <searx.preference>` settings."""
# pylint: disable = too-few-public-methods
import typing as t

View File

@@ -38,7 +38,6 @@ area:
"""
__all__ = ["AnswererInfo", "Answerer", "AnswerStorage"]

View File

@@ -13,7 +13,6 @@ from dataclasses import dataclass
from searx.utils import load_module
from searx.result_types.answer import BaseAnswer
_default = pathlib.Path(__file__).parent
log: logging.Logger = logging.getLogger("searx.answerers")

View File

@@ -11,7 +11,7 @@ from urllib.parse import urlencode
import lxml.etree
import lxml.html
from curl_cffi.requests.exceptions import RequestException
from httpx import HTTPError
from searx import settings
from searx.engines import (
@@ -63,7 +63,7 @@ def bing(query: str, _sxng_locale: str) -> list[str]:
base_url = "https://www.bing.com/AS/Suggestions?"
# cvid has to be a 32 character long string consisting of numbers and uppsercase characters
cvid = ''.join(random.choices(string.ascii_uppercase + string.digits, k=32))
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}), enable_http3=True)
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}))
results: list[str] = []
if response.ok:
@@ -83,7 +83,7 @@ def brave(query: str, _sxng_locale: str) -> list[str]:
url = 'https://search.brave.com/api/suggest?'
url += urlencode({'q': query})
country = 'all'
kwargs = {'cookies': {'country': country}, 'enable_http3': True}
kwargs = {'cookies': {'country': country}}
resp = get(url, **kwargs)
results: list[str] = []
@@ -127,17 +127,18 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
def google_complete(query: str, sxng_locale: str) -> list[str]:
"""Autocomplete from Google. Supports Google's languages
"""Autocomplete from Google. Supports Google's languages and subdomains
(:py:obj:`searx.engines.google.get_google_info`) by using the async REST
API::
https://www.google.com/complete/search?{args}
https://{subdomain}/complete/search?{args}
"""
data = ENGINE_TRAITS.get("google") or {}
traits = EngineTraits(**data)
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
url = 'https://{subdomain}/complete/search?{args}'
args = urlencode(
{
'q': query,
@@ -147,7 +148,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
)
results: list[str] = []
resp = get('https://www.google.com/complete/search?' + args, enable_http3=True)
resp = get(url.format(subdomain=google_info['subdomain'], args=args))
if resp and resp.ok:
json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
data = json.loads(json_txt)
@@ -418,5 +419,5 @@ def search_autocomplete(backend_name: str, query: str, sxng_locale: str) -> list
return []
try:
return backend(query, sxng_locale)
except (RequestException, SearxEngineResponseException):
except (HTTPError, SearxEngineResponseException):
return []

View File

@@ -5,7 +5,6 @@ Implementations used for bot detection.
"""
__all__ = ["init", "dump_request", "get_network", "too_many_requests", "ProxyFix"]

View File

@@ -182,7 +182,7 @@ class Config:
if default is UNSET:
raise KeyError(name)
return default
(modulename, name) = str(fqn).rsplit('.', 1)
modulename, name = str(fqn).rsplit('.', 1)
m = __import__(modulename, {}, {}, [name], 0)
return getattr(m, name)

View File

@@ -13,7 +13,6 @@ Accept_ header ..
"""
from ipaddress import (
IPv4Network,
IPv6Network,

View File

@@ -14,7 +14,6 @@ bot if the Accept-Encoding_ header ..
"""
from ipaddress import (
IPv4Network,
IPv6Network,

View File

@@ -11,7 +11,6 @@ if the Accept-Language_ header is unset.
"""
from ipaddress import (
IPv4Network,
IPv6Network,

View File

@@ -11,7 +11,6 @@ the Connection_ header is set to ``close``.
"""
from ipaddress import (
IPv4Network,
IPv6Network,

View File

@@ -20,6 +20,7 @@ Metadata`_. A request is filtered out in case of:
"""
# pylint: disable=unused-argument

View File

@@ -12,7 +12,6 @@ the User-Agent_ header is unset or matches the regular expression
"""
import re
from ipaddress import (
IPv4Network,
@@ -25,7 +24,6 @@ import flask
from . import config
from ._helpers import too_many_requests
USER_AGENT = (
r'('
+ r'unknown'

View File

@@ -55,7 +55,6 @@ from ._helpers import (
logger,
)
logger = logger.getChild('ip_limit')
BURST_WINDOW = 20

View File

@@ -23,6 +23,7 @@ The ``ip_lists`` method implements :py:obj:`block-list <block_ip>` and
]
"""
# pylint: disable=unused-argument

View File

@@ -1,6 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Implementation of a middleware to determine the real IP of an HTTP request
(:py:obj:`flask.request.remote_addr`) behind a proxy chain."""
# pylint: disable=too-many-branches

View File

@@ -1,7 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Providing a Valkey database for the botdetection methods."""
import valkey
__all__ = ["set_valkey_client", "get_valkey_client"]

View File

@@ -1,5 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Implementations needed for a branding of SearXNG."""
# pylint: disable=too-few-public-methods
# Struct fields aren't discovered in Python 3.14

View File

@@ -465,7 +465,7 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
# Check if value is expired. It's possible that it's expired but has not
# yet been automatically deleted by the periodic maintenance
(value, expire) = row
value, expire = row
now = time.time()
if expire < now:
# The record is deleted during the maintenance interval. Deleting

View File

@@ -3,7 +3,6 @@
import warnings
# limiter backward compatibility
# ------------------------------

View File

@@ -4,6 +4,7 @@
make data.all
"""
# pylint: disable=invalid-name
__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
@@ -32,13 +33,6 @@ class WikiDataUnitType(t.TypedDict):
to_si_factor: float
WikiDataPropertyNameType = str | dict[str, str]
"""Name of a Wikidata property. Can be either the plain name or a dictionary of
language code to property name, e.g. ``{"en": "Date of birth"}``."""
WikiDataPropertiesType = dict[str, WikiDataPropertyNameType]
"""Dictionary from wikidata property ID to property name."""
class LocalesType(t.TypedDict):
"""Data structure of an item in ``locales.json``"""
@@ -48,7 +42,6 @@ class LocalesType(t.TypedDict):
USER_AGENTS: UserAgentType
WIKIDATA_UNITS: dict[str, WikiDataUnitType]
WIKIDATA_PROPERTIES: WikiDataPropertiesType
TRACKER_PATTERNS: TrackerPatternsDB
LOCALES: LocalesType
CURRENCIES: CurrenciesDB
@@ -60,12 +53,11 @@ ENGINE_DESCRIPTIONS: dict[str, dict[str, t.Any]]
ENGINE_TRAITS: dict[str, dict[str, t.Any]]
lazy_globals: dict[str, t.Any] = {
lazy_globals = {
"CURRENCIES": CurrenciesDB(),
"USER_AGENTS": None,
"EXTERNAL_URLS": None,
"WIKIDATA_UNITS": None,
"WIKIDATA_PROPERTIES": None,
"EXTERNAL_BANGS": None,
"OSM_KEYS_TAGS": None,
"ENGINE_DESCRIPTIONS": None,
@@ -78,7 +70,6 @@ data_json_files = {
"USER_AGENTS": "useragents.json",
"EXTERNAL_URLS": "external_urls.json",
"WIKIDATA_UNITS": "wikidata_units.json",
"WIKIDATA_PROPERTIES": "wikidata_properties.json",
"EXTERNAL_BANGS": "external_bangs.json",
"OSM_KEYS_TAGS": "osm_keys_tags.json",
"ENGINE_DESCRIPTIONS": "engine_descriptions.json",

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Simple implementation to store TrackerPatterns data in a SQL database."""
# pylint: disable=too-many-branches
import typing as t
@@ -10,7 +11,7 @@ import re
from collections.abc import Iterator
from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
from curl_cffi.requests.exceptions import RequestException
from httpx import HTTPError
from searx.data.core import get_cache, log
from searx.network import get as http_get
@@ -28,11 +29,11 @@ class TrackerPatternsDB:
ctx_name = "data_tracker_patterns"
# ClearURL rule lists, the first one that responds HTTP 200 is used
CLEAR_LIST_URL = [
"https://cdn.jsdelivr.net/gh/clearurls/rules@refs/heads/gh-pages/data.minify.json",
"https://rules2.clearurls.xyz/data.minify.json",
# ClearURL rule lists, the first one that responds HTTP 200 is used
"https://rules1.clearurls.xyz/data.minify.json",
"https://rules2.clearurls.xyz/data.minify.json",
"https://raw.githubusercontent.com/ClearURLs/Rules/refs/heads/master/data.min.json",
]
class Fields:
@@ -87,8 +88,8 @@ class TrackerPatternsDB:
try:
resp = http_get(url, timeout=3)
except RequestException as exc:
log.warning("TRACKER_PATTERNS: RequestException while fetching %s: %s", url, exc)
except HTTPError as exc:
log.warning("TRACKER_PATTERNS: HTTPError (%s) occured while fetching %s", url, exc)
continue
if resp.status_code != 200:

View File

@@ -5,7 +5,7 @@
],
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
"versions": [
"154.0",
"153.0"
"153.0",
"152.0"
]
}

File diff suppressed because it is too large Load Diff

View File

@@ -3474,6 +3474,11 @@
"symbol": "mm⁻²",
"to_si_factor": 1e-06
},
"Q136039973": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q1361854": {
"si_name": "Q11570",
"symbol": "dwt",
@@ -5249,11 +5254,6 @@
"symbol": "μA",
"to_si_factor": 1e-06
},
"Q31274648": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q3186734": {
"si_name": "Q3186734",
"symbol": "J/(m³ K)",

View File

@@ -305,7 +305,7 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
region: str = ""
"""For an engine, when there is ``region: ...`` in the YAML settings the engine
does support only this one region:
does support only this one region::
.. code:: yaml
@@ -317,9 +317,6 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
enable_http: bool
"""Enable HTTP (by default only HTTPS is enabled)."""
enable_http3: bool = False
"""Enables the use of HTTP/3 if available"""
shortcut: str
"""Code used to execute bang requests (``!foo``)"""

View File

@@ -6,8 +6,7 @@ from urllib.parse import urlencode
from datetime import datetime
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
from searx.utils import html_to_text
from searx.utils import html_to_text, get_embeded_stream_url
about = {
"website": "https://tv.360kan.com/",
@@ -30,12 +29,12 @@ def request(query, params):
return params
def response(resp) -> EngineResults:
def response(resp):
try:
data = resp.json()
except Exception as e:
raise SearxEngineAPIException(f"Invalid response: {e}") from e
res = EngineResults()
results = []
if "data" not in data or "result" not in data["data"]:
raise SearxEngineAPIException("Invalid response")
@@ -51,15 +50,16 @@ def response(resp) -> EngineResults:
except (ValueError, TypeError):
published_date = None
res.add(
res.types.LegacyResult(
url=entry["play_url"],
title=html_to_text(entry["title"]),
content=html_to_text(entry["description"]),
template='videos.html',
publishedDate=published_date,
thumbnail=entry["cover_img"],
)
results.append(
{
'url': entry["play_url"],
'title': html_to_text(entry["title"]),
'content': html_to_text(entry["description"]),
'template': 'videos.html',
'publishedDate': published_date,
'thumbnail': entry["cover_img"],
"iframe_src": get_embeded_stream_url(entry["play_url"]),
}
)
return res
return results

View File

@@ -26,7 +26,6 @@ categories: list[str]
disabled: bool
display_error_messages: bool
enable_http: bool
enable_http3: bool
engine_type: str
inactive: bool
max_page: int

View File

@@ -187,9 +187,8 @@ def set_loggers(engine: "Engine|types.ModuleType", engine_name: str):
def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]):
# pylint: disable=too-many-branches
# set / update engine attributes from engine_data
# set engine attributes from engine_data
kvargs: dict[str, t.Any]
engine.about = getattr(engine, "about", EngineAbout())
if isinstance(engine.about, EngineAbout):
kvargs = {**msgspec.to_builtins(engine.about), **engine_data.get("about", {})}
else:

View File

@@ -35,7 +35,7 @@ Implementation
==============
"""
import typing as t
from datetime import datetime, timedelta
from urllib.parse import urlencode
@@ -85,7 +85,7 @@ Additional subcategories:
# Do we need support for "free_collection" and "include_stock_enterprise"?
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not categories:
raise ValueError("adobe_stock engine: categories is unset")
@@ -100,9 +100,9 @@ def setup(_: dict[str, t.Any]) -> bool | None:
raise ValueError("adobe_stock engine: adobe_content_types is unset")
if isinstance(adobe_content_types, list):
for content_type in adobe_content_types:
if content_type not in ADOBE_VALID_TYPES:
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % content_type)
for t in adobe_content_types:
if t not in ADOBE_VALID_TYPES:
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % t)
else:
raise ValueError(
"adobe_stock engine: adobe_content_types must be a list of strings not %s" % type(adobe_content_types)

View File

@@ -25,6 +25,7 @@ To use this engine, add an entry similar to the following to your engine list in
https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app
"""
import typing as t
from searx.enginelib import EngineCache

View File

@@ -49,9 +49,6 @@ CACHE: EngineCache
def setup(engine_settings: dict[str, t.Any]) -> bool:
if baidu_category not in ('general', 'images', 'it'):
raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
return True
@@ -68,6 +65,11 @@ def get_image_cookies(headers: dict[str, str]) -> dict[str, str]:
return cookies
def init(_):
if baidu_category not in ('general', 'images', 'it'):
raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
def request(query, params):
page_num = params["pageno"]

View File

@@ -1,5 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""BASE (Scholar publications)"""
from datetime import datetime
import re

View File

@@ -40,7 +40,6 @@ about: dict[str, t.Any] = {
# engine dependent config
categories = ["general", "web"]
safesearch = True
enable_http3 = True
_safesearch_map: dict[int, str] = {
0: "off",
1: "moderate",
@@ -72,21 +71,43 @@ def get_locale_params(engine_region: str | None) -> dict[str, str] | None:
return {"mkt": engine_region}
def override_accept_language(params: "OnlineParams", engine_region: str | None) -> None:
"""Override the ``Accept-Language`` header.
The default header built by :py:class:`~searx.search.processors.online.OnlineProcessor`
appends ``en;q=0.3`` as a fallback language::
Accept-Language: de,de-DE;q=0.7,en;q=0.3
Bing seems to better select the results locale based on the
``Accept-Language`` value header.
This function is shared with :py:mod:`searx.engines.bing_images`,
:py:mod:`searx.engines.bing_news`, and :py:mod:`searx.engines.bing_videos`.
"""
if not engine_region or engine_region == "clear":
return
lang = engine_region.split("-")[0]
params["headers"]["Accept-Language"] = f"{engine_region},{lang};q=0.9"
def request(query: str, params: "OnlineParams"):
"""Assemble a Bing-Web request."""
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
override_accept_language(params, engine_region)
query_params: dict[str, str | int] = {
"q": query,
"adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
}
if engine_region and engine_region != "clear":
lang, _, cc = engine_region.partition("-")
query_params["setlang"] = lang
if cc and cc not in ("us", "cn", "ru"): # bing just sends junk for these
query_params["cc"] = cc
locale_params = get_locale_params(engine_region)
if locale_params:
query_params.update(locale_params)
params["url"] = f"{base_url}/search?{urlencode(query_params)}"

View File

@@ -9,6 +9,7 @@ from lxml import html
from searx.engines.bing import ( # pylint: disable=unused-import
fetch_traits,
get_locale_params,
override_accept_language,
)
# about
@@ -24,7 +25,6 @@ about = {
# engine dependent config
categories = ["images", "web"]
paging = True
enable_http3 = True
safesearch = True
time_range_support = True
time_map = {
@@ -43,6 +43,8 @@ def request(query, params):
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
override_accept_language(params, engine_region)
# build URL query
# - example: https://www.bing.com/images/async?q=foo&async=1&first=1&count=35
query_params = {

View File

@@ -12,7 +12,10 @@ from urllib.parse import urlencode
from lxml import html
from searx.enginelib.traits import EngineTraits
from searx.engines.bing import get_locale_params
from searx.engines.bing import (
get_locale_params,
override_accept_language,
)
from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
# about
@@ -30,7 +33,6 @@ categories = ["news"]
paging = True
"""If go through the pages and there are actually no new results for another
page, then bing returns the results from the last page again."""
enable_http3 = True
time_range_support = True
time_map = {
@@ -51,6 +53,8 @@ def request(query, params):
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
override_accept_language(params, engine_region)
# build URL query
# - example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
page = int(params.get("pageno", 1)) - 1

View File

@@ -9,6 +9,7 @@ from lxml import html
from searx.engines.bing import ( # pylint: disable=unused-import
fetch_traits,
get_locale_params,
override_accept_language,
)
from searx.engines.bing_images import time_map
from searx.utils import eval_xpath, eval_xpath_getindex
@@ -25,7 +26,6 @@ about = {
# engine dependent config
categories = ["videos", "web"]
paging = True
enable_http3 = True
safesearch = True
time_range_support = True
@@ -38,6 +38,8 @@ def request(query, params):
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
override_accept_language(params, engine_region)
# build URL query
# - example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
query_params = {

View File

@@ -45,7 +45,7 @@ CACHE_SESSION_ID_KEY = "session_id_key"
KEYWORD_RE = re.compile(r"\[\/?Keyword\]")
def setup(engine_settings: dict[str, t.Any]) -> bool:
def init(engine_settings: dict[str, t.Any]) -> bool:
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_name=engine_settings["name"])
return True

View File

@@ -135,7 +135,7 @@ from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
extract_text,
get_embedded_stream_url,
get_embeded_stream_url,
js_obj_str_to_json_str,
js_obj_str_to_python,
)
@@ -151,7 +151,6 @@ about = {
base_url = "https://search.brave.com/"
categories = []
enable_http3 = True
brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = "search"
"""Brave supports common web-search, videos, images, news, and goggles search.
@@ -248,13 +247,13 @@ def extract_json_data(text: str) -> dict[str, t.Any]:
# node_ids: [0, 19],
# data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# form: null,
# error: null
# });
text = text[text.index("<script") : text.index("</script")]
if not text:
raise ValueError("can't find JS/JSON data in the given text")
start = text.index("data: [{")
newline = text.index("\n", start)
end = text.rindex("}}]", start, newline)
js_obj_str = "{" + text[start:end] + "}}]}"
end = text.rindex("}}]")
js_obj_str = text[start:end]
js_obj_str = "{" + js_obj_str + "}}]}"
# js_obj_str = js_obj_str.replace("\xa0", "") # remove ASCII for &nbsp;
# js_obj_str = js_obj_str.replace(r"\u003C", "<").replace(r"\u003c", "<") # fix broken HTML tags in strings
json_str = js_obj_str_to_json_str(js_obj_str)
@@ -339,7 +338,7 @@ def _parse_search(resp: SXNG_Response) -> EngineResults:
if len(video_tag):
# In my tests a video tag in the WEB search was most often not a
# video, except the ones from youtube ..
iframe_src = get_embedded_stream_url(url)
iframe_src = get_embeded_stream_url(url)
if iframe_src:
item["iframe_src"] = iframe_src
item["template"] = "videos.html"
@@ -354,14 +353,14 @@ def _parse_news(resp: SXNG_Response) -> EngineResults:
res = EngineResults()
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, "//div[@data-type='news']"):
url = eval_xpath_getindex(result, ".//a/@href", 0, default=None)
for result in eval_xpath_list(dom, "//div[contains(@class, 'results')]//div[@data-type='news']"):
url = eval_xpath_getindex(result, ".//a[contains(@class, 'result-header')]/@href", 0, default=None)
if url is None:
continue
title = eval_xpath_list(result, ".//div[contains(@class, 'title')]")
content = eval_xpath_list(result, ".//div[contains(@class, 'description')]")
thumbnail = eval_xpath_getindex(result, ".//a[contains(@class, 'thumbnail')]//img/@src", 0, default="")
title = eval_xpath_list(result, ".//span[contains(@class, 'snippet-title')]")
content = eval_xpath_list(result, ".//p[contains(@class, 'desc')]")
thumbnail = eval_xpath_getindex(result, ".//div[contains(@class, 'image-wrapper')]//img/@src", 0, default="")
item = res.types.LegacyResult(
template="default.html",
@@ -407,6 +406,9 @@ def _parse_videos(json_resp: dict[str, t.Any]) -> EngineResults:
)
if result["thumbnail"] is not None:
item["thumbnail"] = result["thumbnail"]["src"]
iframe_src = get_embeded_stream_url(result["url"])
if iframe_src:
item["iframe_src"] = iframe_src
res.add(item)

View File

@@ -40,7 +40,7 @@ if t.TYPE_CHECKING:
about = {
"website": "https://api.search.brave.com/",
"wikidata_id": None,
"official_api_documentation": "https://api-dashboard.search.brave.com/api-reference/web/search/get",
"official_api_documentation": "https://api-dashboard.search.brave.com/documentation",
"use_official_api": True,
"require_api_key": True,
"results": "JSON",
@@ -63,10 +63,8 @@ base_url = "https://api.search.brave.com/res/v1/web/search"
time_range_map = {"day": "past_day", "week": "past_week", "month": "past_month", "year": "past_year"}
"""Mapping of SearXNG time ranges to Brave API time ranges."""
max_page = 10
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
"""Initialize the engine."""
if not api_key:
raise SearxEngineAPIException("No API key provided")
@@ -77,7 +75,7 @@ def request(query: str, params: "OnlineParams") -> None:
search_args: dict[str, str | int | None] = {
"q": query,
"count": results_per_page,
"offset": params["pageno"] - 1,
"offset": (params["pageno"] - 1) * results_per_page,
"text_decorations": False,
}
@@ -91,7 +89,6 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = f"{base_url}?{urlencode(search_args)}"
params["headers"]["X-Subscription-Token"] = api_key
params["headers"]["Accept"] = "application/json"
def _extract_published_date(published_date_raw: str):

85
searx/engines/cara.py Normal file
View File

@@ -0,0 +1,85 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=invalid-name
"""Cara_ is a social media and portfolio-sharing platform for artists and art
enthusiasts.
With the widespread use of generative AI, Cara_ decided to build a place that
filters out gen AI images so that people searching for authentic creatives and
images can do so easily.
.. _Cara: https://cara.app/about
"""
from urllib.parse import urlencode
import typing as t
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://cara.app",
"official_api_documentation": None,
"use_official_api": False,
"require_api_key": False,
"results": "JSON",
}
base_url = "https://cara.app"
images_url = "https://images.cara.app"
categories = ["images"]
paging = True
results_per_page = 24
# if using HTTP2, we get blocked immediately
enable_http2 = False
def request(query: str, params: "OnlineParams") -> None:
args = {
"q": query,
"sortBy": "Top",
"take": results_per_page,
"skip": (params["pageno"] - 1) * results_per_page,
}
params["url"] = f"{base_url}/api/search/portfolio-posts?{urlencode(args)}"
def response(resp: "SXNG_Response"):
res = EngineResults()
json_data: list[dict[str, t.Any]] = resp.json()
for result in json_data:
thumbnail, img = None, None
i: dict[str, str]
for i in result["images"]:
if thumbnail is None or i["isCoverImg"]:
thumbnail = i
if img is None or not i["isCoverImg"]:
img = i
if not thumbnail or not img:
continue
res.add(
res.types.LegacyResult(
{
"template": "images.html",
"url": f"{base_url}/post/{result['id']}",
"thumbnail_src": f"{images_url}/{thumbnail['src']}?height=256",
"img_src": f"{images_url}/{img['src']}",
"title": result["title"],
"content": result["content"],
"author": result["name"],
}
)
)
return res

View File

@@ -41,7 +41,7 @@ search_index = "cw22"
<https://www.chatnoir.eu/docs/api-general>`_ for a full list."""
def _obtain_api_key() -> tuple[str, str]:
def _obtain_api_key() -> tuple[str, str, str]:
home_resp = get(base_url)
if not home_resp.ok:
raise SearxEngineAPIException("failed to obtain api key")
@@ -58,9 +58,10 @@ def _obtain_api_key() -> tuple[str, str]:
)
if not token_resp.ok:
raise SearxEngineAPIException("failed to obtain api key")
session_id = token_resp.cookies["sessionid"]
scraped_api_key = token_resp.json()["token"]["token"]
return csrf_token, scraped_api_key
return csrf_token, session_id, scraped_api_key
def request(query: str, params: "OnlineParams"):
@@ -72,7 +73,7 @@ def request(query: str, params: "OnlineParams"):
params["headers"].update(headers)
else:
csrf_token, scraped_api_key = _obtain_api_key()
csrf_token, session_id, scraped_api_key = _obtain_api_key()
headers = {
"Authorization": f"Bearer {scraped_api_key}",
@@ -80,11 +81,10 @@ def request(query: str, params: "OnlineParams"):
}
params["headers"].update(headers)
params["cookies"] = {"csrftoken": csrf_token}
params["cookies"] = {"csrftoken": session_id, "sessionid": session_id}
params["url"] = f"{base_url}/api/v1/_search"
params["method"] = "POST"
params["impersonate"] = "none"
json_data = {
"query": query,

View File

@@ -78,7 +78,7 @@ time_range_dict = {'day': '24h', 'week': '1w', 'month': '1m', 'year': '1y'}
base_url = "https://www.chinaso.com"
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if chinaso_news_source not in t.get_args(ChinasoNewsSourceType):
raise ValueError(f"Unsupported news source: {chinaso_news_source}")

View File

@@ -74,7 +74,6 @@ Implementations
===============
"""
import typing as t
import re
from os.path import expanduser, isabs, realpath, commonprefix
from shlex import split as shlex_split
@@ -84,7 +83,6 @@ from threading import Thread
from searx import logger
from searx.result_types import EngineResults
engine_type = 'offline'
paging = True
command = []
@@ -101,7 +99,7 @@ _command_logger = logger.getChild('command')
_compiled_parse_regex = {}
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
check_parsing_options(engine_settings)
if 'command' not in engine_settings:

View File

@@ -141,13 +141,12 @@ def response(resp: "SXNG_Response") -> EngineResults:
if name:
authors.add(name)
tag = result.get("fieldOfStudy")
res.add(
res.types.Paper(
title=result.get("title"),
url=url,
content=result.get("fullText", "") or "",
tags=[tag] if tag else [],
tags=result.get("fieldOfStudy", []),
publishedDate=published_date,
type=result.get("documentType", "") or "",
authors=authors,

View File

@@ -1,5 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Docker Hub (IT)"""
# pylint: disable=use-dict-literal
from urllib.parse import urlencode

View File

@@ -8,9 +8,6 @@ import typing as t
from datetime import datetime, timezone
import html
from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineAPIException
from searx.network import post
from searx.utils import format_duration, html_to_text, humanize_number
from searx.result_types import EngineResults
@@ -38,36 +35,15 @@ dogpile_categ = "search"
base_url = "https://www.dogpile.com"
safe_search_map = {0: "none", 1: "moderate", 2: "heavy"}
CACHE: EngineCache
"""Cache for the API token from dogpile"""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if dogpile_categ not in ("search", "images", "videos", "news"):
raise ValueError("invalid search type: %s" % dogpile_categ)
global CACHE # pylint: disable=global-statement
CACHE = EngineCache("dogpile") # one token for images/videos/news
return True
def _obtain_token() -> str:
token = CACHE.get("token")
if token:
return token
resp = post(f"{base_url}/api/token/refresh", headers={"Origin": base_url}, cookies={"dp_api_token": "1"})
if not resp.ok:
raise SearxEngineAPIException("failed to obtain dogpile token")
token = resp.json()["token"]
CACHE.set("token", token, expire=240) # 300s ttl
return token
def request(query: str, params: "OnlineParams"):
params["url"] = f"{base_url}/api/{dogpile_categ}"
params["headers"]["Origin"] = base_url
params["cookies"]["dp_api_token"] = "1"
if dogpile_categ != "search": # web doesnt need token
params["headers"]["x-dogpile-token"] = _obtain_token()
params["method"] = "POST"
params["json"] = {"q": query, "qadf": safe_search_map[params["safesearch"]], "page": params["pageno"]}

View File

@@ -164,6 +164,7 @@ Terms / phrases that you keep coming across:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Accept-Language
"""
# pylint: disable=global-statement
import json

View File

@@ -12,6 +12,7 @@ least we could not find out how language support should work. It seems that
most of the features are based on English terms.
"""
import typing as t
from urllib.parse import urlencode, urlparse, urljoin

View File

@@ -10,8 +10,7 @@ from datetime import datetime
from urllib.parse import urlencode
from urllib.parse import quote_plus
from searx.result_types import EngineResults, MainResult, LegacyResult, Image
from searx.utils import html_to_text, gen_useragent, extr
from searx.utils import get_embeded_stream_url, html_to_text, gen_useragent, extr
from searx.network import get # see https://github.com/searxng/searxng/issues/762
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
@@ -48,7 +47,7 @@ _HTTP_User_Agent: str = gen_useragent()
send_accept_language_header = False
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings: dict[str, t.Any]):
if engine_settings["ddg_category"] not in ["images", "videos", "news"]:
raise ValueError(f"Unsupported DuckDuckGo category: {engine_settings['ddg_category']}")
@@ -98,7 +97,6 @@ def request(query: str, params: "OnlineParams") -> None:
# The vqd value is generated from the query and the UA header. To be able to
# reuse the vqd value, the UA header must be static.
headers["User-Agent"] = _HTTP_User_Agent
params["impersonate"] = "none"
vqd = get_vqd(query=query, params=params) or fetch_vqd(query=query, params=params)
headers["Accept"] = "*/*"
@@ -150,51 +148,54 @@ def request(query: str, params: "OnlineParams") -> None:
def _image_result(result):
return Image(
url=result['url'],
title=result['title'],
content='',
thumbnail_src=result['thumbnail'],
img_src=result['image'],
resolution='%s x %s' % (result['width'], result['height']),
source=result['source'],
)
return {
'template': 'images.html',
'url': result['url'],
'title': result['title'],
'content': '',
'thumbnail_src': result['thumbnail'],
'img_src': result['image'],
'resolution': '%s x %s' % (result['width'], result['height']),
'source': result['source'],
}
def _video_result(result):
return LegacyResult(
template='videos.html',
url=result['content'],
title=result['title'],
content=result['description'],
thumbnail=result['images'].get('small') or result['images'].get('medium'),
source=result['provider'],
length=result['duration'],
metadata=result.get('uploader'),
)
return {
'template': 'videos.html',
'url': result['content'],
'title': result['title'],
'content': result['description'],
'thumbnail': result['images'].get('small') or result['images'].get('medium'),
'iframe_src': get_embeded_stream_url(result['content']),
'source': result['provider'],
'length': result['duration'],
'metadata': result.get('uploader'),
}
def _news_result(result):
return MainResult(
url=result['url'],
title=result['title'],
content=html_to_text(result['excerpt']),
publishedDate=datetime.fromtimestamp(result['date']),
)
return {
'url': result['url'],
'title': result['title'],
'content': html_to_text(result['excerpt']),
'source': result['source'],
'publishedDate': datetime.fromtimestamp(result['date']),
}
def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
def response(resp):
results = []
res_json = resp.json()
for result in res_json['results']:
if ddg_category == 'images':
res.add(_image_result(result))
results.append(_image_result(result))
elif ddg_category == 'videos':
res.add(_video_result(result))
results.append(_video_result(result))
elif ddg_category == 'news':
res.add(_news_result(result))
results.append(_news_result(result))
else:
raise ValueError(f"Invalid duckduckgo category: {ddg_category}")
return res
return results

View File

@@ -17,7 +17,6 @@ from searx.result_types import EngineResults
from searx.extended_types import SXNG_Response
from searx import weather
about = {
"website": 'https://duckduckgo.com/',
"wikidata_id": 'Q12805',
@@ -109,19 +108,7 @@ def response(resp: SXNG_Response):
json_data = loads(resp.text[resp.text.find('\n') + 1 : resp.text.rfind('\n') - 2])
location = json_data.get("location")
if not location:
return res
metadata = json_data.get("weatherAlerts", {}).get("metadata", {})
geoloc = weather.GeoLocation(
name=location,
latitude=metadata.get("latitude"),
longitude=metadata.get("longitude"),
elevation=0,
country_code=metadata.get("language").split("-")[-1],
timezone=json_data.get("location"),
)
geoloc = weather.GeoLocation.by_query(resp.search_params["query"])
weather_answer = EngineResults.types.WeatherAnswer(
current=_weather_data(geoloc, json_data["currentWeather"]),

View File

@@ -14,12 +14,11 @@ can't build it ourselves and must scrape it from the HTML pages.
"""
import typing as t
import re
from urllib.parse import quote_plus, urljoin
from urllib.parse import quote_plus
from lxml import html
from searx.utils import html_to_text, extract_text, eval_xpath
from searx.utils import html_to_text, gen_useragent, extract_text, eval_xpath
from searx.result_types import EngineResults
from searx.enginelib import EngineCache
from searx.network import get
@@ -39,6 +38,7 @@ about = {
# engine dependent config
categories = ["general"]
paging = True
_HTTP_User_Agent: str = gen_useragent()
base_url = "https://duckduckgo.com"
@@ -73,8 +73,6 @@ def _fetch_first_page_link(
resp = get(
url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
headers=headers,
impersonate="firefox",
default_headers=False,
timeout=2,
)
@@ -98,43 +96,6 @@ def _cache_key(query: str, pageno: int) -> str:
return f"nextpage_url|{query}|{pageno}"
def _solve_jsa(resp: "SXNG_Response") -> "SXNG_Response":
"""Duckduckgo sometimes issues a challenge instead of json."""
# length that a real browser would report for where the broken snippet is
html_len = {
"<p><div></p><p></div": 32,
"<li><div></li><li></div": 29,
"<div><div></div><div></div": 33,
"<br><div></br><br></div": 23,
}
js = resp.text or ""
jsa_match = re.search(r"let jsa = (\d+);.*?DDG\.deep\.initialize\('([^']+)'", js, re.S)
if not jsa_match:
return resp
js_functions = dict(re.findall(r"let (\w+) = function\(num\) \{([^}]*)\};", js))
jsa = int(jsa_match.group(1))
try:
for name in re.findall(r"jsa = (\w+)\(jsa\);", js):
body = js_functions[name]
mul = re.search(r"num \* (\d+)", body)
jsa = jsa * int(mul.group(1)) if mul else jsa + html_len[re.search(r"`([^`]+)`", body).group(1)]
except (KeyError, AttributeError):
return resp
params = resp.search_params
follow = get(
urljoin("https://links.duckduckgo.com", jsa_match.group(2) + str(jsa)),
headers=params["headers"],
impersonate="firefox",
default_headers=False,
)
follow.search_params = params
return follow
def request(query: str, params: "OnlineParams") -> None:
if len(query) >= 500:
@@ -142,15 +103,25 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = None
return
# firefox TLS only
params["impersonate"] = "firefox"
params["default_headers"] = False
headers = params["headers"]
# The vqd value is generated from the query and the UA header. To be able
# to reuse the vqd value, the UA header must be static.
headers["User-Agent"] = _HTTP_User_Agent
headers["Accept"] = "*/*"
headers["Referer"] = f"{base_url}/"
headers["Host"] = "duckduckgo.com"
# Sec-Fetch headers are required to not get blocked when sending a Firefox user agent
headers["Sec-Fetch-Dest"] = "script"
headers["Sec-Fetch-Mode"] = "no-cors"
headers["Sec-Fetch-Site"] = "same-site"
api_url = ""
if params["pageno"] > 1:
api_url = CACHE.get(_cache_key(query, params["pageno"]))
else:
api_url = _fetch_first_page_link(query, params["headers"])
api_url = _fetch_first_page_link(query, headers)
if not api_url:
params["url"] = None
@@ -158,27 +129,14 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
# loads as a script
headers = params["headers"]
headers["Accept"] = "*/*"
headers["Sec-Fetch-Dest"] = "script"
headers["Sec-Fetch-Mode"] = "no-cors"
headers["Sec-Fetch-Site"] = "same-site"
headers["Referer"] = f"{base_url}/"
# TODO: support safesearch, timerange and engine traits # pylint:disable=fixme
def response(resp: "SXNG_Response"):
res = EngineResults()
res_json = resp.json()
# check if ddg returns a challenge
# e.g. 'site:github.com searxng'
if "let jsa =" in (resp.text or ""):
resp = _solve_jsa(resp)
results = resp.json()["results"]
for result in results:
for result in res_json["results"]:
if "u" not in result:
continue
@@ -186,13 +144,13 @@ def response(resp: "SXNG_Response"):
res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
)
if results:
next_page_path = results[-1].get("n")
if next_page_path:
CACHE.set(
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
base_url + next_page_path,
expire=60 * 60,
)
# link to next page
next_page_path = res_json["results"][-1].get("n")
if next_page_path:
CACHE.set(
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
base_url + next_page_path,
expire=60 * 60,
)
return res

View File

@@ -2,7 +2,6 @@
# pylint: disable=invalid-name
"""Dummy Offline"""
# about
about = {
"wikidata_id": None,

View File

@@ -41,7 +41,6 @@ authentication configured to read from ``my-index`` index.
"""
import typing as t
from json import loads, dumps
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
@@ -69,7 +68,7 @@ show_metadata = False
page_size = 10
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
if 'query_type' in engine_settings and engine_settings['query_type'] not in _available_query_types:
raise ValueError('unsupported query type', engine_settings['query_type'])

View File

@@ -1,150 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""`Europe PMC`_ provides comprehensive access to life sciences literature from
trusted sources. With Europe PMC you can search and read millions of
publications, preprints and other documents enriched with links to supporting
data, reviews, protocols, and other relevant resources.
.. _Europe PMC: https://europepmc.org/
Configuration
=============
.. code:: yaml
- name: europepmc
engine: europepmc
shortcut: epmc
Implementations
===============
"""
import typing as t
from datetime import datetime
from urllib.parse import urlencode
from dateutil.parser import isoparse
from searx.enginelib import EngineCache
from searx.result_types import EngineResults
from searx.utils import html_to_text
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://europepmc.org/",
"wikidata_id": "Q5412157",
"official_api_documentation": "https://europepmc.org/RestfulWebService",
"use_official_api": True,
"require_api_key": False,
"results": "JSON",
}
categories = ["science", "scientific publications"]
paging = True
# engine dependent config
search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
article_url = "https://europepmc.org/article/"
page_size = 20
CACHE: EngineCache
"""Cache for storing the pagination cursor."""
def setup(engine_settings: dict[str, t.Any]):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
def _cache_key(query: str, page: int) -> str:
return f"{query}|{page}"
def request(query: str, params: "OnlineParams") -> None:
args = {
"query": query,
"format": "json",
"resultType": "core",
"pageSize": page_size,
}
if params["pageno"] > 1:
if cursor := CACHE.get(_cache_key(query, params["pageno"])):
args["cursorMark"] = cursor
else:
# no cached cursor for that page
params["url"] = None
return
params["url"] = f"{search_url}?{urlencode(args)}"
def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
json_resp = resp.json()
# store pagination cursor for loading next pages in cache
if next_cursor := json_resp.get("nextCursorMark"):
next_page = resp.search_params["pageno"] + 1
query = resp.search_params["query"]
CACHE.set(_cache_key(query, next_page), next_cursor)
all_results = json_resp.get("resultList", {}).get("result", [])
for item in all_results:
source = item.get("source", "")
identifier = item.get("id", "")
url = f"{article_url}{source}/{identifier}" if source and identifier else ""
journal_info: dict[str, t.Any] = item.get("journalInfo", {})
journal: dict[str, t.Any] = journal_info.get("journal", {})
res.add(
res.types.Paper(
url=url,
title=html_to_text(item.get("title", "")),
content=html_to_text(item.get("abstractText", "")),
journal=journal.get("title", ""),
issn=[journal.get("issn", "")],
authors=_get_authors(item),
doi=item.get("doi", ""),
publishedDate=_get_published_date(item),
type=", ".join((item.get("pubTypeList", {})).get("pubType", [])),
pdf_url=_get_pdf_url(item),
html_url=url,
)
)
return res
def _get_authors(item: dict[str, t.Any]) -> list:
"""Extract the list of authors from the item."""
if authors := item.get("authorString", None):
authors = [author.strip().rstrip(".") for author in authors.split(",") if author.strip()]
else:
authors = []
return authors
def _get_pdf_url(item: dict[str, t.Any]) -> str:
"""Extract the PDF URL in case it is open access."""
for url_info in (item.get("fullTextUrlList", {})).get("fullTextUrl", []):
if url_info.get("documentStyle") == "pdf" and url_info.get("availabilityCode") == "OA":
return url_info.get("url", "")
return ""
def _get_published_date(item: dict[str, t.Any]) -> datetime | None:
"""Extract the published date from the item and convert it to a datetime object."""
if unformatted_date := item.get("firstPublicationDate"):
return isoparse(unformatted_date)
return None

View File

@@ -78,7 +78,7 @@ content_max_characters: int = 500
"""Maximum characters for the requested content."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not api_key:
raise SearxEngineAPIException("No API key provided")
if not 1 <= results_per_page <= 100:

View File

@@ -47,7 +47,7 @@ CACHE_VALID_DURATION = 30 * 24 * 3600 # one month, same as website
"""Duration how long settings cookies are valid."""
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings: dict[str, t.Any]):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])

View File

@@ -38,7 +38,6 @@ Implementation
"""
import typing as t
from urllib.parse import urlencode
from dateutil import parser
@@ -78,7 +77,7 @@ page_size: int = 10
"""Maximum number of results per page (default 10)."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not base_url:
raise ValueError('gitea engine: base_url is unset')

View File

@@ -65,7 +65,6 @@ code lines are just relabeled (starting from 1) and appended (a disjoint set of
code blocks in a single file might be returned from the API).
"""
import typing as t
from urllib.parse import urlencode

View File

@@ -9,15 +9,12 @@ engines:
- :ref:`google scholar engine`
- :ref:`google autocomplete`
This implementation uses Nokia user agents to request an XML layout from Google.
The normal web version requires executing JavaScript to load the results and
therefore is currently not used here. See `Google discussion`_ for more
information on that topic.
.. _Google discussion: https://github.com/searxng/searxng/issues/6359
"""
import random
import re
import string
import time
import typing as t
from urllib.parse import unquote, urlencode
@@ -47,16 +44,16 @@ about = {
"official_api_documentation": "https://developers.google.com/custom-search/",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["general", "web"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
@@ -67,23 +64,38 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
# Filter results. 0: None, 1: Moderate, 2: Strict
filter_mapping = {0: "off", 1: "medium", 2: "high"}
# https://github.com/searxng/searxng/issues/6359
nokia_useragents = (
"Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
)
# specific xpath variables
# ------------------------
# Suggestions are links placed in a *card-section*, we extract only the text
# from the links not the links itself.
suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
_arcid_range = string.ascii_letters + string.digits + "_-"
_arcid_random: tuple[str, int] | None = None
def ui_async(start: int) -> str:
"""Format of the response from UI's async request.
- ``arc_id:<...>,use_ac:true,_fmt:prog``
The arc_id is random generated every hour.
"""
global _arcid_random # pylint: disable=global-statement
use_ac = "use_ac:true"
# _fmt:html returns a HTTP 500 when user search for celebrities like
# '!google natasha allegri' or '!google chris evans'
_fmt = "_fmt:prog"
# create a new random arc_id every hour
if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
_arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
return ",".join([arc_id, use_ac, _fmt])
def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
@@ -115,11 +127,19 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
A instance of :py:obj:`babel.core.Locale` build from the
``searxng_locale`` value.
subdomain:
Google subdomain :py:obj:`google_domains` that fits to the country
code.
params:
Py-Dictionary with additional request arguments (can be passed to
:py:func:`urllib.parse.urlencode`).
- ``hl`` parameter: specifies the interface language of user interface.
- ``lr`` parameter: restricts search results to documents written in
a particular language.
- ``cr`` parameter: restricts search results to documents
originating in a particular country.
- ``ie`` parameter: sets the character encoding scheme that should
be used to interpret the query string ('utf8').
- ``oe`` parameter: sets the character encoding scheme that should
@@ -136,6 +156,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val: dict[str, t.Any] = {
"language": None,
"country": None,
"subdomain": None,
"params": {},
"headers": {},
"cookies": {},
@@ -148,7 +169,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
except babel.core.UnknownLocaleError:
locale = None
eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
lang_code = eng_lang.split("_")[-1] # lang_zh-TW --> zh-TW / lang_en --> en
country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
@@ -163,6 +184,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val["language"] = eng_lang
ret_val["country"] = country
ret_val["locale"] = locale
ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
# hl parameter:
# The hl parameter specifies the interface language (host language) of
@@ -201,11 +223,9 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
# specify a region (country) only if a region is given in the selected
# locale --> https://github.com/searxng/searxng/issues/2672
if country is not None:
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
# gl parameter: (mandatory by Google News)
# The gl parameter value is a two-letter country code. For WebSearch
@@ -280,78 +300,88 @@ def detect_google_sorry(resp: "SXNG_Response"):
raise SearxEngineCaptchaException()
def unwrap_google_url(raw_url: str) -> str:
# remove redirector from url
if raw_url.startswith("/url?q="):
return unquote(raw_url[7:].split("&sa=U")[0])
return raw_url
def wml_dom(resp: "SXNG_Response"):
detect_google_sorry(resp)
text = resp.text
if text.lstrip().startswith("<?xml"):
text = text.split("?>", 1)[-1]
return html.fromstring(text)
def google_request(
query: str,
params: "OnlineParams",
extra_args: dict[str, t.Any] | None = None,
*,
eng_traits: EngineTraits | None = None,
use_time_range: bool = True,
use_safesearch: bool = True,
safesearch_map: dict[int, str] | None = None,
use_locales: bool = True,
) -> None:
google_info = get_google_info(params, eng_traits or traits)
if not use_locales:
google_info["params"].pop("lr")
google_info["params"].pop("cr")
start = (params["pageno"] - 1) * 10
args: dict[str, t.Any] = {
"q": query,
"sca_esv": "1",
**google_info["params"],
**(extra_args or {}),
}
if start:
args["start"] = start
if use_time_range and params["time_range"] in time_range_dict:
args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
if use_safesearch and params["safesearch"]:
args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
params["headers"]["User-Agent"] = random.choice(nokia_useragents)
params["impersonate"] = "chrome99_android"
def request(query: str, params: "OnlineParams") -> None:
google_request(query, params)
"""Google search request"""
# pylint: disable=line-too-long
start = (params["pageno"] - 1) * 10
google_info = get_google_info(params, traits)
# https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search"
+ "?"
+ urlencode(
{
"q": query,
**google_info["params"],
"filter": "0",
"start": start,
# 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
# 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
# 'cs' : 1,
# 'sa': 'N',
# 'yv': 3,
# 'prmd': 'vin',
# 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
# 'sa': 'N',
# 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
# formally known as use_mobile_ui
# "asearch": "arc",
# "async": str_async,
}
)
)
if params["time_range"] in time_range_dict:
query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
if params["safesearch"]:
query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
# regex match to get image map that is found inside the returned javascript:
# (function(){var s='...';var i=['...'] ...}
RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
def parse_url_images(text: str):
data_image_map = {}
for image_url, img_id in RE_DATA_IMAGE.findall(text):
data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def response(resp: "SXNG_Response"):
"""Get response from google's search request"""
# pylint: disable=too-many-branches, too-many-statements
detect_google_sorry(resp)
data_image_map = parse_url_images(resp.text)
results = EngineResults()
dom = wml_dom(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
# parse results
for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
# pylint: disable=too-many-nested-blocks
try:
title_tag = eval_xpath_getindex(
result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
)
title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
if title_tag is None:
# this not one of the common google results *section*
logger.debug("ignoring item from the result_xpath list: missing title")
continue
title = extract_text(title_tag)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
raw_url = result.get("href")
if raw_url is None:
logger.debug(
'ignoring item from the result_xpath list: missing url of title "%s"',
@@ -359,19 +389,30 @@ def response(resp: "SXNG_Response") -> EngineResults:
)
continue
url = unwrap_google_url(raw_url)
content = extract_text(
eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
)
thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
results.add(
results.types.MainResult(
url=url,
title=title or "",
content=content or "",
thumbnail=thumbnail or "",
)
)
if raw_url.startswith('/url?q='):
url = unquote(raw_url[7:].split("&sa=U")[0]) # remove the google redirector
else:
url = raw_url
content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
for item in content_nodes:
for script in item.xpath(".//script"):
script.getparent().remove(script)
content = extract_text(content_nodes[0])
# Images that are NOT the favicon
xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
thumbnail = None
if xpath_image is not None:
thumbnail = xpath_image.get("src")
if thumbnail.startswith("data:image"):
img_id = xpath_image.get("id")
if img_id:
thumbnail = data_image_map.get(img_id)
results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
except Exception as e: # pylint: disable=broad-except
logger.error(e, exc_info=True)
@@ -379,8 +420,10 @@ def response(resp: "SXNG_Response") -> EngineResults:
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
# append suggestion
results.append({"suggestion": extract_text(suggestion)})
# return results
return results
@@ -413,12 +456,14 @@ skip_countries = [
]
def fetch_traits(engine_traits: EngineTraits):
def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
"""Fetch languages from Google."""
# pylint: disable=import-outside-toplevel, too-many-branches
from searx.network import get # see https://github.com/searxng/searxng/issues/762
engine_traits.custom["supported_domains"] = {}
resp = get("https://www.google.com/preferences", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google preferences is not OK.")
@@ -469,3 +514,22 @@ def fetch_traits(engine_traits: EngineTraits):
# alias regions
engine_traits.regions["zh-CN"] = "HK"
# supported domains
if add_domains:
resp = get("https://www.google.com/supported_domains", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google supported domains is not OK.")
for domain in resp.text.split():
domain = domain.strip()
if not domain or domain in [
".google.com",
]:
continue
region = domain.split(".")[-1].upper()
engine_traits.custom["supported_domains"][region] = "www" + domain
if region == "HK":
# There is no google.cn, we use .com.hk for zh-CN
engine_traits.custom["supported_domains"]["CN"] = "www" + domain

View File

@@ -30,7 +30,6 @@ about = {
categories = ["general", "web"]
paging = True
enable_http3 = True
max_page = 5
page_size = 20
time_range_support = True
@@ -96,11 +95,12 @@ def request(query: str, params: "OnlineParams") -> None:
token = _cse_token()
google_info = get_google_info(params, traits)
info: dict[str, str] = google_info["params"]
args = {
"rsz": "filtered_cse",
"num": str(page_size),
"hl": google_info["params"]["hl"],
"hl": info["hl"],
"cselibv": token["cselibv"],
"cx": CX,
"q": query,
@@ -114,6 +114,10 @@ def request(query: str, params: "OnlineParams") -> None:
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
args["sort"] = f"date:r:{start_date}:{end_date}"
if info.get("lr"):
args["lr"] = info["lr"]
if info.get("cr"):
args["cr"] = info["cr"]
if google_info["country"] not in (None, "ZZ"):
args["gl"] = google_info["country"]
if token["exp"]:

View File

@@ -1,76 +1,122 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Images: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Images engine using the internal
Google API used by the Google Go Android app.
import typing as t
from urllib.parse import parse_qs, unquote, urlparse
This internal API offer results in
- JSON (``_fmt:json``)
- Protobuf_ (``_fmt:pb``)
- Protobuf_ compressed? (``_fmt:pc``)
- HTML (``_fmt:html``)
- Protobuf_ encoded in JSON (``_fmt:jspb``).
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
"""
from urllib.parse import urlencode
from json import loads
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, wml_dom
from searx.result_types import EngineResults
from searx.utils import eval_xpath_list
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import (
get_google_info,
time_range_dict,
detect_google_sorry,
)
# about
about = {
"website": "https://images.google.com",
"wikidata_id": "Q521550",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://images.google.com',
"wikidata_id": 'Q521550',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'JSON',
}
# engine dependent config
categories = ["images", "web"]
categories = ['images', 'web']
paging = True
enable_http3 = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
safesearch = True
filter_mapping = {0: "images", 1: "active", 2: "active"}
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "isch"},
eng_traits=traits,
safesearch_map=filter_mapping,
use_locales=False,
def request(query, params):
"""Google-Image search request"""
google_info = get_google_info(params, traits)
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ '?'
+ urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
# don't urlencode this because wildly different AND bad results
# pagination uses Zero-based numbering
+ f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if params['safesearch']:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
# this ua will allow getting ~50 results instead of 10. #1641
params['headers']['User-Agent'] = (
'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
)
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
dom = wml_dom(resp)
return params
for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
qs = parse_qs(urlparse(link.get("href", "")).query)
img_src = qs.get("imgurl", [""])[0]
url = qs.get("imgrefurl", [""])[0]
if not img_src or not url:
continue
width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
tbnid = qs.get("tbnid", [""])[0]
results.add(
results.types.Image(
url=url,
title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
img_src=img_src,
thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
resolution=f"{width} x {height}" if width and height else "",
)
)
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
json_start = resp.text.find('{"ischj":')
json_data = loads(resp.text[json_start:])
for item in json_data["ischj"].get("metadata", []):
result_item = {
'url': item["result"]["referrer_url"],
'title': item["result"]["page_title"],
'content': item["text_in_grid"]["snippet"],
'source': item["result"]["site_title"],
'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
'img_src': item["original_image"]["url"],
'thumbnail_src': item["thumbnail"]["url"],
'template': 'images.html',
}
author = item["result"].get('iptc', {}).get('creator')
if author:
result_item['author'] = ', '.join(author)
copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
if copyright_notice:
result_item['source'] += ' | ' + copyright_notice
freshness_date = item["result"].get("freshness_date")
if freshness_date:
result_item['source'] += ' | ' + freshness_date
file_size = item.get('gsa', {}).get('file_size')
if file_size:
result_item['source'] += ' (%s)' % file_size
results.append(result_item)
return results

View File

@@ -1,91 +1,325 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google News: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google News engine.
Google News has a different region handling compared to Google WEB.
- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
- the hl_ argument has to be set correctly (and different to Google WEB)
- the gl_ argument is mandatory
If one of this argument is not set correctly, the request is redirected to
CONSENT dialog::
https://consent.google.com/m?continue=
The google news API ignores some parameters from the common :ref:`google API`:
- num_ : the number of search results is ignored / there is no paging all
results for a query term are in the first response.
- save_ : is ignored / Google-News results are always *SafeSearch*
.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
"""
import typing as t
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
import json
import base64
from urllib.parse import urlencode
from lxml import html
import babel
from searx import locales
from searx.utils import (
eval_xpath_getindex,
eval_xpath,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
)
from searx.engines.google import fetch_traits as _fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
detect_google_sorry,
)
from searx.enginelib.traits import EngineTraits
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
# about
about = {
"website": "https://www.google.com",
"website": "https://news.google.com",
"wikidata_id": "Q12020",
"official_api_documentation": "https://developers.google.com/custom-search",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["news"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
paging = False
time_range_support = False
language_support = True
safesearch = False
# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
# False here.
#
# safesearch : results are identical for safesearch=0 and safesearch=2
safesearch = True
base_url: str = "https://news.google.com"
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "nws"},
eng_traits=traits,
use_time_range=False,
use_safesearch=False,
use_locales=False,
"""Google-News search request"""
sxng_locale = params.get("searxng_locale", "en-US")
ceid: str = locales.get_engine_locale(
sxng_locale, traits.custom["ceid"], default="US:en"
) # pyright: ignore[reportAssignmentType]
google_info = get_google_info(params, traits)
google_info["subdomain"] = "news.google.com" # google news has only one domain
ceid_region, ceid_lang = ceid.split(":")
ceid_lang, ceid_suffix = (
ceid_lang.split(":")
+ [
"",
]
)[:2]
google_info["params"]["hl"] = ceid_lang
if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
if ceid_region.lower() == ceid_lang:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
elif ceid_region.lower() != ceid_lang:
if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
google_info["params"]["hl"] = ceid_lang
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
google_info["params"]["gl"] = ceid_region
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search?"
+ urlencode(
{"q": query, **google_info["params"]},
)
# ceid includes a ':' character which must not be urlencoded
+ ("&ceid=%s" % ceid)
)
def _span_text(link, css_class: str):
return extract_text(
eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
allow_none=True,
)
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
seen = set()
for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
href = link.get("href")
if not href:
"""Get response from google's search request"""
res = EngineResults()
detect_google_sorry(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
if not url:
continue
if url.startswith("./"):
url = base_url + url[1:]
# The real URL is often encoded in the "jslog" attribute
jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
# Try to extract the real URL from jslog
real_url: str | None = None
if jslog:
# jslog format is usually: "95014; 5:<base64>; track:click,vis". We
# want the second part (index 1) after splitting by ";"
parts: list[str] = jslog.split(";")
if len(parts) > 1:
b64_data: str = parts[1].split(":")[-1].strip()
# Pad base64 if necessary
b64_data += "=" * (-len(b64_data) % 4)
decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
# The URL is typically the last element in the decoded array
if (
isinstance(decoded_data, list)
and isinstance(decoded_data[-1], str)
and decoded_data[-1].startswith("http")
):
real_url = decoded_data[-1]
if real_url:
url = real_url
else:
logger.error(f"no real-url found: {url}")
continue
url = unwrap_google_url(href)
if url in seen or "google.com/search" in url:
continue
title = extract_text(eval_xpath(result, "./h4")) or ""
title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
if not title:
continue
# The pub_date is mostly a string like 'yesterday', not a real timezone
# date or time. Therefore we can't use publishedDate and place the
# *pub* sting into the content.
source = _span_text(link, "dXDvrc")
pub_date = _span_text(link, "YVIcad")
thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
pub_date = extract_text(eval_xpath(result, ".//time"))
pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
content = " / ".join([x for x in [pub_origin, pub_date] if x])
seen.add(url)
results.add(
results.types.MainResult(
thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
if thumbnail and thumbnail.startswith("/"):
thumbnail = base_url + thumbnail
res.add(
res.types.MainResult(
url=url,
title=title,
content=" / ".join(x for x in [source, pub_date] if x),
thumbnail=thumbnail or "",
content=content,
thumbnail=thumbnail,
)
)
return results
return res
ceid_list = [
"AE:ar",
"AR:es-419",
"AT:de",
"AU:en",
"BD:bn",
"BE:fr",
"BE:nl",
"BG:bg",
"BR:pt-419",
"BW:en",
"CA:en",
"CA:fr",
"CH:de",
"CH:fr",
"CL:es-419",
"CN:zh-Hans",
"CO:es-419",
"CU:es-419",
"CZ:cs",
"DE:de",
"EE:et",
"EG:ar",
"ES:ca",
"ES:es",
"ET:en",
"FI:fi",
"FR:fr",
"GB:en",
"GH:en",
"GR:el",
"HK:zh-Hant",
"HU:hu",
"ID:en",
"ID:id",
"IE:en",
"IL:en",
"IL:he",
"IN:bn",
"IN:en",
"IN:gu",
"IN:hi",
"IN:ml",
"IN:mr",
"IN:pa",
"IN:ta",
"IN:te",
"IT:it",
"JP:ja",
"KE:en",
"KR:ko",
"LB:ar",
"LT:lt",
"LV:en",
"LV:lv",
"MA:fr",
"MY:en",
"MY:ms",
"NA:en",
"NG:en",
"NL:nl",
"NO:no",
"NZ:en",
"PH:en",
"PK:en",
"PL:pl",
"RO:ro",
"RS:sr",
"RU:ru",
"SA:ar",
"SE:sv",
"SG:en",
"SI:sl",
"SK:sk",
"SN:fr",
"TH:th",
"TR:tr",
"TZ:en",
"UA:ru",
"UA:uk",
"UG:en",
"US:en",
"VN:vi",
"ZA:en",
"ZW:en",
]
"""List of region/language combinations supported by Google News. Values of the
``ceid`` argument of the Google News REST API."""
_skip_values = [
"ET:en", # english (ethiopia)
"ID:en", # english (indonesia)
"LV:en", # english (latvia)
]
_ceid_locale_map = {"NO:no": "nb-NO"}
def fetch_traits(engine_traits: EngineTraits):
_fetch_traits(engine_traits, add_domains=False)
engine_traits.custom["ceid"] = {}
for ceid in ceid_list:
if ceid in _skip_values:
continue
region, lang = ceid.split(":")
x = lang.split("-")
if len(x) > 1:
if x[1] not in ["Hant", "Hans"]:
lang = x[0]
sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
try:
locale = babel.Locale.parse(sxng_locale, sep="-")
except babel.UnknownLocaleError:
print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
continue
engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid

View File

@@ -20,7 +20,6 @@ about = {
}
play_categ = None # apps|movies
enable_http3 = True
base_url = 'https://play.google.com'
search_url = base_url + "/store/search?{query}&c={play_categ}"

View File

@@ -27,7 +27,7 @@ import typing as t
from urllib.parse import urlencode
from datetime import datetime
from lxml import html
from curl_cffi.requests.exceptions import TooManyRedirects
import httpx
from searx.utils import (
eval_xpath,
@@ -63,7 +63,6 @@ about = {
# engine dependent config
categories = ["science", "scientific publications"]
paging = True
enable_http3 = True
max_page = 50
"""`Google max 50 pages`_
@@ -78,6 +77,8 @@ def request(query: str, params: "OnlineParams") -> None:
"""Google-Scholar search request"""
google_info = get_google_info(params, traits)
# subdomain is: scholar.google.xy
google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
args = {
"q": query,
@@ -88,7 +89,7 @@ def request(query: str, params: "OnlineParams") -> None:
}
args.update(time_range_args(params))
params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
@@ -103,7 +104,7 @@ def response(resp: "SXNG_Response") -> EngineResults: # pylint: disable=too-man
raise SearxEngineAccessDeniedException(
message="google_scholar: unusual traffic detected",
)
raise TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
raise httpx.TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
res = EngineResults()
dom = html.fromstring(resp.text)

View File

@@ -1,85 +1,186 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Videos: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Videos engine.
import typing as t
.. admonition:: Content-Security-Policy (CSP)
This engine needs to allow images from the `data URLs`_ (prefixed with the
``data:`` scheme)::
Header set Content-Security-Policy "img-src 'self' data: ;"
.. _data URLs:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
"""
import re
from urllib.parse import urlencode, urlparse, parse_qs, unquote
from lxml import html
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
parse_duration_string,
)
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
time_range_dict,
filter_mapping,
suggestion_xpath,
detect_google_sorry,
ui_async,
)
from searx.utils import get_embeded_stream_url
# about
about = {
"website": "https://www.google.com",
"wikidata_id": "Q219885",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://www.google.com',
"wikidata_id": 'Q219885',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'HTML',
}
# engine dependent config
categories = ["videos", "web"]
categories = ['videos', 'web']
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
language_support = True
time_range_support = True
safesearch = True
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "vid"},
eng_traits=traits,
use_locales=False,
# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
def parse_data_images(text: str):
data_image_map = {}
for img_id, data_image in RE_DATA_IMAGE.findall(text):
end_pos = data_image.rfind("=")
if end_pos > 0:
data_image = data_image[: end_pos + 1]
data_image_map[img_id] = data_image
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def request(query, params):
"""Google-Video search request"""
google_info = get_google_info(params, traits)
start = (params['pageno'] - 1) * 10
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ "?"
+ urlencode(
{
'q': query,
'tbm': "vid",
'start': start,
**google_info['params'],
'asearch': 'arc',
'async': ui_async(start),
}
)
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if 'safesearch' in params:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
return params
for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
data_image_map = parse_data_images(resp.text)
# convert the text to dom
dom = html.fromstring(resp.text)
result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
# parse results
for result in result_divs:
title = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
allow_none=True,
)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
if not title or not raw_url:
continue
url = unwrap_google_url(raw_url)
thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
if "/default.jpg" in thumbnail:
thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
length = None
for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
length = parse_duration_string(extract_text(span) or "")
if length:
break
results.add(
results.types.MainResult(
url=url,
title=title,
thumbnail=thumbnail,
length=length,
template="videos.html",
)
url = eval_xpath_getindex(
result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
)
if url and url.startswith('/url?q='):
url = unquote(url[7:].split('&sa=U')[0])
content = extract_text(
eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
)
pub_info = extract_text(
eval_xpath_getindex(
result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
),
allow_none=True,
)
# Broader XPath to find any <img> element
thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
duration = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
)
video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
# Fallback for video_id from URL if not found via XPath
if not video_id and url and 'youtube.com' in url:
parsed_url = urlparse(url)
video_id = parse_qs(parsed_url.query).get('v', [None])[0]
# Handle thumbnail
if thumbnail and thumbnail.startswith('data:image'):
img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
if img_id and img_id in data_image_map:
thumbnail = data_image_map[img_id]
else:
thumbnail = None
if not thumbnail and video_id:
thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
# Handle video embed URL
embed_url = None
if video_id:
embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
elif url:
embed_url = get_embeded_stream_url(url)
# Only append results with valid title and url
if title and url:
results.append(
{
'url': url,
'title': title,
'content': content or '',
'author': pub_info,
'thumbnail': thumbnail,
'length': duration,
'iframe_src': embed_url,
'template': 'videos.html',
}
)
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.append({'suggestion': extract_text(suggestion)})
return results

127
searx/engines/heexy.py Normal file
View File

@@ -0,0 +1,127 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Heexy_ is a minimalist search engine that focuses on privacy.
Although it also supports news and videos, these are not implemented here
because they usually return no result to very few irrelevant ones.
It seems to use Bing internally, as the image thumbnails are loaded from Bing.
.. _Heexy: https://docs.heexy.org/introduction
"""
from urllib.parse import urlencode
import typing as t
from searx.enginelib import EngineCache
from searx.network import get
from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
from searx.result_types import EngineResults
from searx.utils import gen_useragent
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://heexy.org",
"wikidata_id": None,
"official_api_documentation": None,
"use_official_api": False,
"require_api_key": False,
"results": "JSON",
}
paging = True
safesearch = True
categories = ["general"]
heexy_categ = "web"
"""Category to search in. Can be either "web" or "image"."""
base_url = "https://heexy.org"
api_url = "https://seapi.heexy.org"
safe_search_map = {0: "off", 1: "on", 2: "on"}
CACHE: EngineCache
"""Cache for storing the ``X-Data-Cacheft`` token (acts like an API key)."""
def setup(engine_settings: dict[str, t.Any]) -> bool:
global CACHE # pylint: disable=global-statement
if heexy_categ not in ("web", "image"):
raise ValueError("invalid search category: %s" % heexy_categ)
CACHE = EngineCache(engine_settings["name"])
return True
def _get_api_token(query: str) -> str:
"""The API token is independent of the search query. We just need any query
to obtain it initially, and don't hardcode it here to decrease chances of
getting blocked. The token must be passed as ``X-Data-Cacheft`` header."""
cached_token: str = CACHE.get("token")
if cached_token:
return cached_token
resp = get(
f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent(), "Accept-Language": "en-US,en:q=0.9"}
)
if not resp.ok:
raise SearxEngineAPIException("failed to obtain request token: invalid response code")
token = resp.cookies["cacheft"]
if not token:
raise SearxEngineAPIException("failed to obtain request token: no token found")
CACHE.set("token", token, expire=3 * 60)
return token
def request(query: str, params: "OnlineParams") -> None:
args = {
"q": query,
"page": params["pageno"],
"safe": safe_search_map[params["safesearch"]],
}
if params["searxng_locale"] != "all":
args["lang"] = params["searxng_locale"].split("-")[0]
params["url"] = f"{api_url}/search/{heexy_categ}?{urlencode(args)}"
params["headers"]["Origin"] = api_url
params["cookies"]["cacheft"] = _get_api_token(query)
def response(resp: "SXNG_Response"):
res = EngineResults()
json_resp = resp.json()
if not json_resp["success"]:
raise SearxEngineAccessDeniedException()
result: dict[str, str]
for result in json_resp["results"]:
if heexy_categ == "web":
res.add(
res.types.MainResult(
url=result["url"],
title=result["title"],
content=result["description"],
)
)
elif heexy_categ == "image":
res.add(
res.types.Image(
title=result["description"],
url=result["url"],
thumbnail_src=result["image"],
img_src=result["rawImage"],
)
)
return res

View File

@@ -4,7 +4,6 @@
from urllib.parse import urlencode
from dateutil import parser
about = {
# pylint: disable=line-too-long
"website": "https://hex.pm/",

View File

@@ -33,7 +33,6 @@ Implementations
"""
import typing as t
from urllib.parse import urlencode
from datetime import datetime
@@ -63,7 +62,7 @@ huggingface_endpoint = 'models'
"""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if huggingface_endpoint not in ('datasets', 'models', 'spaces'):
raise SearxEngineAPIException(f"Unsupported Hugging Face endpoint: {huggingface_endpoint}")

View File

@@ -8,7 +8,6 @@ No public instance offer a public API now
"""
import typing as t
import time
import random
from urllib.parse import quote_plus, urlparse
@@ -35,7 +34,7 @@ time_range_support = True
base_url: list[str] | str = []
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not base_url:
raise ValueError("missing invidious base_url")

View File

@@ -108,14 +108,12 @@ def get_infobox(alt_forms, result_url, definitions):
infobox_content.append(f'<p><i>Other forms:</i> {", ".join(alt_forms[1:])}</p>')
# definitions
infobox_content.append(
'''
infobox_content.append('''
<small><a href="https://www.edrdg.org/wiki/index.php/JMdict-EDICT_Dictionary_Project">JMdict</a>
and <a href="https://www.edrdg.org/enamdict/enamdict_doc.html">JMnedict</a>
by <a href="https://www.edrdg.org/edrdg/licence.html">EDRDG</a>, CC BY-SA 3.0.</small>
<ul>
'''
)
''')
for pos, engdef, extra in definitions:
if pos == 'Wikipedia definition':
infobox_content.append('</ul><small>Wikipedia, CC BY-SA 3.0.</small><ul>')

View File

@@ -87,7 +87,7 @@ api_key = ""
"""Kagi API key. Required for using this engine."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not api_key:
raise ValueError("api_key is required for using kagi")

View File

@@ -37,17 +37,21 @@ about = {
}
categories = []
safesearch = True
safeseach = True
base_url = "https://luxxle.com"
luxxle_categ = "search"
"""Supported categories: "search", "news", "images", "videos"."""
# otherwise all requests get blocked (http2-fingerprinted probably)
enable_http2 = False
safe_search_map = {0: "Off", 1: "Moderate", 2: "Strict"}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if luxxle_categ not in ("search", "images", "videos", "news"):
raise ValueError("invalid luxxle category: %s" % luxxle_categ)

View File

@@ -10,7 +10,7 @@ Lofgren .
.. _marginalia filters:
Marginalia Filters
==================
=================
Custom filters enable server-side customization of Marginalia search results.
Filter definitions are written in XML and scoped to an API key. Filters can
@@ -82,7 +82,7 @@ api_key = None
https://about.marginalia-search.com/article/api/
"""
filter_name: str = ""
filter_name: str | None = None
"""The name of the custom filter to apply to each search."""
@@ -164,7 +164,7 @@ def response(resp: SXNG_Response):
return res
def setup(_: dict[str, t.Any]) -> bool:
def init(_: dict[str, t.Any]):
if not api_key:
logger.error("missing api_key: see https://about.marginalia-search.com/article/api")

View File

@@ -26,8 +26,6 @@ Implementations
"""
import typing as t
try:
import mariadb # pyright: ignore [reportMissingImports]
except ImportError:
@@ -62,17 +60,15 @@ paging = True
_connection = None
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
global _connection # pylint: disable=global-statement
if 'query_str' not in engine_settings:
raise ValueError('query_str cannot be empty')
if not engine_settings['query_str'].lower().startswith('select '):
raise ValueError('only SELECT query is supported')
def init(_):
global _connection # pylint: disable=global-statement
_connection = mariadb.connect(database=database, user=username, password=password, host=host, port=port)

View File

@@ -35,7 +35,7 @@ Here is a simple example to query a Meilisearch instance:
"""
# pylint: disable=global-statement
import typing as t
from json import dumps
from searx.result_types import EngineResults
from searx.extended_types import SXNG_Response
@@ -49,7 +49,7 @@ categories = ['general']
paging = True
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if index == '':
raise ValueError('index cannot be empty')

View File

@@ -1,7 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Mojeek (general, images, news)"""
import typing as t
from datetime import datetime
from urllib.parse import urlencode
@@ -52,7 +51,7 @@ region_param = "arc"
_delta_kwargs = {"day": "days", "week": "weeks", "month": "months", "year": "years"}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if search_type not in ("", "images", "news"):
raise ValueError(f"Invalid search type {search_type}")

View File

@@ -49,7 +49,6 @@ except ImportError:
from searx.result_types import EngineResults
engine_type = 'offline'
# mongodb connection variables

View File

@@ -19,7 +19,6 @@ Implementation
==============
"""
import typing as t
from urllib.parse import quote_plus
about = {
@@ -38,7 +37,7 @@ matrix_url = "https://matrix.to"
page_size = 20
def setup(_: dict[str, t.Any]) -> bool | None:
def init(engine_settings): # pylint: disable=unused-argument
"""The ``base_url`` must be set in the configuration, if ``base_url`` is not
set, a :py:obj:`ValueError` is raised during initialization.

View File

@@ -25,8 +25,6 @@ Implementations
"""
import typing as t
from searx.result_types import EngineResults
try:
@@ -62,17 +60,15 @@ paging = True
_connection = None
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
global _connection # pylint: disable=global-statement
if 'query_str' not in engine_settings:
raise ValueError('query_str cannot be empty')
if not engine_settings['query_str'].lower().startswith('select '):
raise ValueError('only SELECT query is supported')
def init(_):
global _connection # pylint: disable=global-statement
_connection = mysql.connector.connect(
database=database,
user=username,

View File

@@ -2,8 +2,6 @@
# pylint: disable=line-too-long
"""Naver for SearXNG"""
import typing as t
from urllib.parse import urlencode
from lxml import html
@@ -18,6 +16,7 @@ from searx.utils import (
html_to_text,
parse_duration_string,
js_obj_str_to_python,
get_embeded_stream_url,
)
# engine metadata
@@ -68,7 +67,7 @@ naver_category_dict = {
}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if naver_category not in ('general', 'images', 'news', 'videos'):
raise SearxEngineAPIException(f"Unsupported category: {naver_category}")
@@ -194,7 +193,7 @@ def parse_news(data):
def parse_videos(data):
res = EngineResults()
results = []
dom = html.fromstring(data)
@@ -213,14 +212,15 @@ def parse_videos(data):
except (ValueError, TypeError):
pass
res.add(
res.types.LegacyResult(
template="videos.html",
title=extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
url=url,
thumbnail=thumbnail,
length=length,
)
results.append(
{
"template": "videos.html",
"title": extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
"url": url,
"thumbnail": thumbnail,
"length": length,
"iframe_src": get_embeded_stream_url(url),
}
)
return res
return results

View File

@@ -6,14 +6,10 @@
from json import loads
import typing as t
from urllib.parse import urlencode
from lxml import html
from searx.exceptions import SearxEngineAPIException
from searx.extended_types import SXNG_Response
from searx.network import get
from searx.result_types import EngineResults
from searx.utils import eval_xpath, extract_text
if t.TYPE_CHECKING:
from searx.enginelib.traits import EngineTraits
@@ -29,33 +25,18 @@ about = {
"results": "JSON",
}
paging = False
enable_http3 = True
base_url = "https://neosearch.org"
categories = ["general"]
def _obtain_xsrf_token() -> str:
resp = get(base_url)
doc = html.fromstring(resp.text)
xsrf_token = extract_text(eval_xpath(doc, "//meta[@name='xsrf-token']/@content"))
if not xsrf_token:
raise SearxEngineAPIException("failed to obtain xsrf token")
return xsrf_token
paging = False
def request(query: str, params: "OnlineParams"):
params["url"] = f"{base_url}/search"
params["headers"]["X-XSRF-TOKEN"] = _obtain_xsrf_token()
params["method"] = "POST"
args = {"q": query, "generate": "auto"}
countrycode = params["searxng_locale"].split("-")[-1].upper()
if countrycode in traits.custom["countrycodes"]:
args["loc"] = countrycode
params["json"] = args
params["url"] = f"{base_url}/search?{urlencode(args)}"
def response(resp: "SXNG_Response") -> EngineResults:
@@ -86,6 +67,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
def fetch_traits(engine_traits: "EngineTraits") -> None:
# pylint: disable=import-outside-toplevel
from searx.network import get
from searx.utils import extr, js_obj_str_to_python
from babel.core import get_global

Some files were not shown because too many files have changed in this diff Show More