jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2026-09-11 16:56:05 +00:00
Code Issues Projects Releases Wiki Activity

Compare commits

base: jandieman:dependabot/pip/master/minor-16b941ef60
Branches Tags
jandieman:master jandieman:gh-pages jandieman:dependabot/pip/master/minor-16b941ef60 jandieman:dependabot/pip/master/black-26.5.1 jandieman:translations jandieman:update_data_update_engine_descriptions.py
..
compare: jandieman:de9cceb4bd6880c299359ee664540ad64544a233
Branches Tags
jandieman:gh-pages jandieman:master jandieman:dependabot/pip/master/minor-16b941ef60 jandieman:dependabot/pip/master/black-26.5.1 jandieman:translations jandieman:update_data_update_engine_descriptions.py

1 Commits
dependabot ... de9cceb4bd

Author SHA1 Message Date
searxng-bot
de9cceb4bd [l10n] update translations from Weblate 
930f4d95f - 2026-08-21 - return42 <return42@noreply.codeberg.org>
45e05f3a5 - 2026-08-21 - return42 <return42@noreply.codeberg.org>
f97f47265 - 2026-08-21 - return42 <return42@noreply.codeberg.org>
c5de4638c - 2026-08-21 - return42 <return42@noreply.codeberg.org>
02aac306b - 2026-08-21 - return42 <return42@noreply.codeberg.org>
f88e28cd0 - 2026-08-14 - Mooo <mooo@noreply.codeberg.org>
 2026-08-21 20:56:43 +00:00
149 changed files with 8066 additions and 30521 deletions
Expand all files Collapse all files

9
.github/workflows/container.yml vendored
View File

@@ -62,7 +62,7 @@ jobs:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Setup QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -105,9 +105,8 @@ jobs:
arch: amd64
- runner: ubuntu-26.04-arm
arch: arm64
# FIXME: https://github.com/searxng/searxng/pull/6655#issuecomment-5550293085
# - runner: ubuntu-26.04-arm
# arch: armv7
- runner: ubuntu-26.04-arm
arch: armv7
steps:
- name: Login to GHCR
@@ -118,7 +117,7 @@ jobs:
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Setup QEMU
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

2
.github/workflows/data-update.yml vendored
View File

@@ -31,7 +31,7 @@ jobs:
- update_external_bangs.py
- update_firefox_version.py
- update_engine_traits.py
- update_wikidata.py
- update_wikidata_units.py
- update_engine_descriptions.py
permissions:

10
client/simple/generated/pygments.less
View File

@@ -2,7 +2,7 @@
/*
this file is generated automatically by searxng_extra/update/update_pygments.py
using pygments version 2.21.0:
using pygments version 2.20.0:
./manage templates.simple.pygments
*/
@@ -114,14 +114,14 @@
.gd { color: #FF4689 } /* Generic.Deleted */
.ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
.ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.gr { color: #FF4689 } /* Generic.Error */
.gr { color: #F8F8F2 } /* Generic.Error */
.gh { color: #F8F8F2 } /* Generic.Heading */
.gi { color: #A6E22E } /* Generic.Inserted */
.go { color: #66D9EF } /* Generic.Output */
.gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
.gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
.gu { color: #959077 } /* Generic.Subheading */
.gt { color: #66D9EF } /* Generic.Traceback */
.gt { color: #F8F8F2 } /* Generic.Traceback */
.kc { color: #66D9EF } /* Keyword.Constant */
.kd { color: #66D9EF } /* Keyword.Declaration */
.kn { color: #FF4689 } /* Keyword.Namespace */
@@ -132,7 +132,7 @@
.m { color: #AE81FF } /* Literal.Number */
.s { color: #E6DB74 } /* Literal.String */
.na { color: #A6E22E } /* Name.Attribute */
.nb { color: #A6E22E } /* Name.Builtin */
.nb { color: #F8F8F2 } /* Name.Builtin */
.nc { color: #A6E22E } /* Name.Class */
.no { color: #66D9EF } /* Name.Constant */
.nd { color: #A6E22E } /* Name.Decorator */
@@ -166,7 +166,7 @@
.sr { color: #E6DB74 } /* Literal.String.Regex */
.s1 { color: #E6DB74 } /* Literal.String.Single */
.ss { color: #E6DB74 } /* Literal.String.Symbol */
.bp { color: #A6E22E } /* Name.Builtin.Pseudo */
.bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
.fm { color: #A6E22E } /* Name.Function.Magic */
.vc { color: #F8F8F2 } /* Name.Variable.Class */
.vg { color: #F8F8F2 } /* Name.Variable.Global */

903
client/simple/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

12
client/simple/package.json
View File

@@ -29,21 +29,21 @@
"swiped-events": "1.2.0"
},
"devDependencies": {
"@biomejs/biome": "2.5.11",
"@types/node": "^26.4.0",
"@biomejs/biome": "2.5.7",
"@types/node": "^26.2.0",
"browserslist": "^4.28.8",
"browserslist-to-esbuild": "^2.1.1",
"edge.js": "^6.5.1",
"less": "^4.9.0",
"less": "^4.8.1",
"mathjs": "^15.2.0",
"sharp": "~0.35.4",
"sharp": "~0.35.3",
"sort-package-json": "^4.0.0",
"stylelint": "^17.14.1",
"stylelint-config-standard-less": "^4.1.0",
"stylelint-prettier": "^5.0.3",
"svgo": "^4.1.0",
"svgo": "^4.0.2",
"typescript": "~7.0.2",
"vite": "^8.2.2",
"vite": "^8.2.1",
"vite-bundle-analyzer": "^1.3.9"
}
}

23
docs/admin/settings/settings_engines.rst
View File

@@ -58,9 +58,10 @@ engine is shown. Most of the options have a default value or even are optional.
# overwrite values from section 'outgoing:'
enable_http2: false
enable_http3: false
retries: 1
max_connections: 100
max_keepalive_connections: 10
keepalive_expiry: 5.0
using_tor_proxy: false
proxies:
http:
@@ -162,16 +163,6 @@ engine is shown. Most of the options have a default value or even are optional.
``enable_http`` : optional
Enable HTTP for this engine (by default only HTTPS is enabled).
``enable_http3`` : optional
Use HTTP/3 (falls back to HTTP/2). Default ``false``.
Ignored when a proxy is set.
.. hint::
HTTP/3 places demands on the IP infrastructure that are not met in every
environment. Enable this option only if you are aware of these requirements
and the extent to which they are met.
``retry_on_http_error`` : optional
Retry request on some HTTP status code.
@@ -188,12 +179,20 @@ engine is shown. Most of the options have a default value or even are optional.
Using tor proxy (``true``) or not (``false``) for this engine. The default is
taken from ``using_tor_proxy`` of the :ref:`settings outgoing`.
.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
``max_keepalive_connection#s`` :
`Pool limit configuration`_, overwrites value ``pool_maxsize`` from
:ref:`settings outgoing` for this engine.
``max_connections`` :
`Pool limit configuration`_, overwrites value ``pool_connections`` from
:ref:`settings outgoing` for this engine.
``keepalive_expiry`` :
`Pool limit configuration`_, overwrites value ``keepalive_expiry`` from
:ref:`settings outgoing` for this engine.
.. _private engines:

51
docs/admin/settings/settings_outgoing.rst
View File

@@ -12,12 +12,20 @@ Communication with search engines.
request_timeout: 2.0 # default timeout in seconds, can be override by engine
max_request_timeout: 10.0 # the maximum timeout in seconds
useragent_suffix: "" # information like an email address to the administrator
pool_connections: 100 # Maximum number of concurrent connections (default: 100)
enable_http2: true # Enables the use of HTTP2
pool_connections: 100 # Maximum number of allowable connections, or null
# for no limits. The default is 100.
pool_maxsize: 10 # Number of allowable keep-alive connections, or null
# to always allow. The default is 10.
enable_http2: true # See https://www.python-httpx.org/http2/
# uncomment below section if you want to use a custom server certificate
# see https://www.python-httpx.org/advanced/#changing-the-verification-defaults
# and https://www.python-httpx.org/compatibility/#ssl-configuration
# verify: ~/.mitmproxy/mitmproxy-ca-cert.cer
#
# uncomment below section if you want to use a proxy
# uncomment below section if you want to use a proxyq see: SOCKS proxies
# https://2.python-requests.org/en/latest/user/advanced/#proxies
# are also supported: see
# https://2.python-requests.org/en/latest/user/advanced/#socks
#
# proxies:
# all://:
@@ -38,26 +46,30 @@ Communication with search engines.
timeout to load). Can be override by ``timeout`` in the :ref:`settings engines`.
``useragent_suffix`` :
Suffix to add when an engine's User-Agent is set via searxng_useragent().
Contact info here may be useful to avoid an engine blocking you.
Suffix to the user-agent SearXNG uses to send requests to others engines. If an
engine wish to block you, a contact info here may be useful to avoid that.
.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
``pool_maxsize``:
Number of allowable keep-alive connections, or ``null`` to always allow. The
default is 10. See ``max_keepalive_connections`` `Pool limit configuration`_.
``pool_connections`` :
Maximum number of concurrent connections. The default is 100.
See ``max_clients`` `Pool limit configuration`_.
Maximum number of allowable connections, or ``null`` # for no limits. The
default is 100. See ``max_connections`` `Pool limit configuration`_.
.. _curl_cffi proxies: https://curl-cffi.readthedocs.io/en/latest/quick_start.html
``keepalive_expiry`` :
Number of seconds to keep a connection in the pool. By default 5.0 seconds.
See ``keepalive_expiry`` `Pool limit configuration`_.
.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying
``proxies`` :
Define one or more proxies you wish to use, see `curl_cffi proxies`_.
Define one or more proxies you wish to use, see `httpx proxies`_.
If there are more than one proxy for one protocol (http, https),
requests to the engines are distributed in a round-robin fashion.
HTTP, HTTPS, SOCKS4, SOCKS5 and SOCKS5h proxies are supported
(``http://``, ``https://``, ``socks4://``, ``socks5://``, ``socks5h://``). You should
use ``socks5h://`` when using Tor so hostnames are resolved by the proxy.
``source_ips`` :
If you use multiple network interfaces, define from which IP the requests must
be made. Example:
@@ -75,15 +87,18 @@ Communication with search engines.
different proxy and source ip.
``enable_http2`` :
Enable by default (HTTP/2). Set to ``false`` to force HTTP/1.1.
HTTP/3 is opt-in per engine (``enable_http3``).
Enable by default. Set to ``false`` to disable HTTP/2.
.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults
.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration
``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR``
HTTPS verification uses the OS's trust store by default.
Set a path to use a custom CA file.
Allow to specify a path to certificate.
see `httpx verification defaults`_.
In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and
``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.
see `httpx ssl configuration`_.
``max_redirects`` :
30 by default. Maximum redirect before it is an error.

4
docs/dev/engines/engine_overview.rst
View File

@@ -143,7 +143,7 @@ parameters with default value can be redefined for special purposes.
data dict ``{}``
cookies dict ``{}``
verify bool ``True``
headers.User-Agent str ``''``
headers.User-Agent str a random User-Agent
category str current category, like ``'general'``
safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict)
time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year``
@@ -229,8 +229,6 @@ following parameters can be used to specify a search request:
max_redirects int maximum redirects, hard limit
soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine
raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300
impersonate str curl_cffi impersonate target (default: chrome, none to disable)
curl_options dict Any extra libcurl options for the request
=================== =========== ==========================================================================

8
docs/dev/engines/online/cara.rst Normal file
View File

@@ -0,0 +1,8 @@
.. _cara engine:
===========
Cara Images
===========
.. automodule:: searx.engines.cara
:members:

8
docs/dev/engines/online/europepmc.rst
View File

@@ -1,8 +0,0 @@
.. _europepmc engine:
==========
Europe PMC
==========
.. automodule:: searx.engines.europepmc
:members:

4
docs/dev/quickstart.rst
View File

@@ -80,8 +80,8 @@ same environment, here are a few examples::
# to test one of the update scripts
(dev.env)$ searxng_extra/update/update_engine_traits.py --help
# to test the update of the wikidata units and property names
(dev.env)$ searxng_extra/update/update_wikidata.py
# to test the update of the wikidata units
(dev.env)$ searxng_extra/update/update_wikidata_units.py
.. sidebar:: further read

2
docs/dev/reST.rst
View File

@@ -286,7 +286,7 @@ content becomes smart.
files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst```
pull request :pull:`4` ``:pull:`4```
patch :patch:`af2cae6` ``:patch:`af2cae6```
PyPi package :pypi:`curl_cffi` ``:pypi:`curl_cffi```
PyPi package :pypi:`httpx` ``:pypi:`httpx```
manual page man :man:`bash` ``:man:`bash```
intersphinx_
--------------------------------------------------------------------------------------------------

6
docs/dev/searxng_extra/update.rst
View File

@@ -90,10 +90,10 @@ Scripts to update static data in :origin:`searx/data/`
:members:
``update_wikidata.py``
``update_wikidata_units.py``
============================
:origin:`[source] <searxng_extra/update/update_wikidata.py>`
:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
.. automodule:: searxng_extra.update.update_wikidata
.. automodule:: searxng_extra.update.update_wikidata_units
:members:

10
requirements-dev.txt
View File

@@ -2,9 +2,9 @@ mock==5.2.0
nose2[coverage_plugin]==0.16.0
cov-core==1.15.0
black==25.9.0
pylint==4.0.8
pylint==4.0.7
splinter==0.21.0
selenium==4.48.0
selenium==4.47.0
Sphinx==8.2.3;python_version <= "3.11"
Sphinx==9.1.0; python_version > "3.11"
sphinx-issues==6.0.0
@@ -18,11 +18,11 @@ myst-parser==5.0.0
linuxdoc==20260504
aiounittest==1.5.0
yamllint==1.38.0
wlc==2.2.0
wlc==2.1.1
coloredlogs==15.0.1
docutils>=0.21.2;python_version <= "3.11"
docutils>=0.22.4; python_version > "3.11"
parameterized==0.9.0
granian[reload]==2.8.2
basedpyright==1.40.0
granian[reload]==2.8.1
basedpyright==1.39.9
types-lxml==2026.2.16

4
requirements-server.txt
View File

@@ -1,2 +1,2 @@
granian==2.8.2
granian[pname]==2.8.2
granian==2.8.1
granian[pname]==2.8.1

10
requirements.txt
View File

@@ -3,15 +3,17 @@ babel==2.18.0
flask-babel==4.0.0
flask==3.1.3
jinja2==3.1.6
lxml==6.1.3
pygments==2.21.0
lxml==6.1.1
pygments==2.20.0
python-dateutil==2.9.0.post0
pyyaml==6.0.3
curl_cffi==0.16.3
httpx[http2]==0.28.1
httpx-socks[asyncio]==0.13.1
sniffio==1.3.1
valkey==6.1.1
markdown-it-py==4.2.0
msgspec==0.21.1
typer==0.27.2
typer==0.27.1
isodate==0.7.2
whitenoise==6.12.0
typing-extensions==4.16.0

15
searx/autocomplete.py
View File

@@ -11,7 +11,7 @@ from urllib.parse import urlencode
import lxml.etree
import lxml.html
from curl_cffi.requests.exceptions import RequestException
from httpx import HTTPError
from searx import settings
from searx.engines import (
@@ -63,7 +63,7 @@ def bing(query: str, _sxng_locale: str) -> list[str]:
base_url = "https://www.bing.com/AS/Suggestions?"
# cvid has to be a 32 character long string consisting of numbers and uppsercase characters
cvid = ''.join(random.choices(string.ascii_uppercase + string.digits, k=32))
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}), enable_http3=True)
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}))
results: list[str] = []
if response.ok:
@@ -83,7 +83,7 @@ def brave(query: str, _sxng_locale: str) -> list[str]:
url = 'https://search.brave.com/api/suggest?'
url += urlencode({'q': query})
country = 'all'
kwargs = {'cookies': {'country': country}, 'enable_http3': True}
kwargs = {'cookies': {'country': country}}
resp = get(url, **kwargs)
results: list[str] = []
@@ -127,17 +127,18 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
def google_complete(query: str, sxng_locale: str) -> list[str]:
"""Autocomplete from Google. Supports Google's languages
"""Autocomplete from Google. Supports Google's languages and subdomains
(:py:obj:`searx.engines.google.get_google_info`) by using the async REST
API::
https://www.google.com/complete/search?{args}
https://{subdomain}/complete/search?{args}
"""
data = ENGINE_TRAITS.get("google") or {}
traits = EngineTraits(**data)
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
url = 'https://{subdomain}/complete/search?{args}'
args = urlencode(
{
'q': query,
@@ -147,7 +148,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
)
results: list[str] = []
resp = get('https://www.google.com/complete/search?' + args, enable_http3=True)
resp = get(url.format(subdomain=google_info['subdomain'], args=args))
if resp and resp.ok:
json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
data = json.loads(json_txt)
@@ -418,5 +419,5 @@ def search_autocomplete(backend_name: str, query: str, sxng_locale: str) -> list
return []
try:
return backend(query, sxng_locale)
except (RequestException, SearxEngineResponseException):
except (HTTPError, SearxEngineResponseException):
return []

12
searx/data/__init__.py
View File

@@ -32,13 +32,6 @@ class WikiDataUnitType(t.TypedDict):
to_si_factor: float
WikiDataPropertyNameType = str | dict[str, str]
"""Name of a Wikidata property. Can be either the plain name or a dictionary of
language code to property name, e.g. ``{"en": "Date of birth"}``."""
WikiDataPropertiesType = dict[str, WikiDataPropertyNameType]
"""Dictionary from wikidata property ID to property name."""
class LocalesType(t.TypedDict):
"""Data structure of an item in ``locales.json``"""
@@ -48,7 +41,6 @@ class LocalesType(t.TypedDict):
USER_AGENTS: UserAgentType
WIKIDATA_UNITS: dict[str, WikiDataUnitType]
WIKIDATA_PROPERTIES: WikiDataPropertiesType
TRACKER_PATTERNS: TrackerPatternsDB
LOCALES: LocalesType
CURRENCIES: CurrenciesDB
@@ -60,12 +52,11 @@ ENGINE_DESCRIPTIONS: dict[str, dict[str, t.Any]]
ENGINE_TRAITS: dict[str, dict[str, t.Any]]
lazy_globals: dict[str, t.Any] = {
lazy_globals = {
"CURRENCIES": CurrenciesDB(),
"USER_AGENTS": None,
"EXTERNAL_URLS": None,
"WIKIDATA_UNITS": None,
"WIKIDATA_PROPERTIES": None,
"EXTERNAL_BANGS": None,
"OSM_KEYS_TAGS": None,
"ENGINE_DESCRIPTIONS": None,
@@ -78,7 +69,6 @@ data_json_files = {
"USER_AGENTS": "useragents.json",
"EXTERNAL_URLS": "external_urls.json",
"WIKIDATA_UNITS": "wikidata_units.json",
"WIKIDATA_PROPERTIES": "wikidata_properties.json",
"EXTERNAL_BANGS": "external_bangs.json",
"OSM_KEYS_TAGS": "osm_keys_tags.json",
"ENGINE_DESCRIPTIONS": "engine_descriptions.json",

430
searx/data/ahmia_blacklist.txt
View File

File diff suppressed because it is too large Load Diff

1234
searx/data/engine_traits.json
View File

File diff suppressed because it is too large Load Diff

7684
searx/data/external_bangs.json
View File

File diff suppressed because it is too large Load Diff

12
searx/data/tracker_patterns.py
View File

@@ -10,7 +10,7 @@ import re
from collections.abc import Iterator
from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
from curl_cffi.requests.exceptions import RequestException
from httpx import HTTPError
from searx.data.core import get_cache, log
from searx.network import get as http_get
@@ -28,11 +28,11 @@ class TrackerPatternsDB:
ctx_name = "data_tracker_patterns"
# ClearURL rule lists, the first one that responds HTTP 200 is used
CLEAR_LIST_URL = [
"https://cdn.jsdelivr.net/gh/clearurls/rules@refs/heads/gh-pages/data.minify.json",
"https://rules2.clearurls.xyz/data.minify.json",
# ClearURL rule lists, the first one that responds HTTP 200 is used
"https://rules1.clearurls.xyz/data.minify.json",
"https://rules2.clearurls.xyz/data.minify.json",
"https://raw.githubusercontent.com/ClearURLs/Rules/refs/heads/master/data.min.json",
]
class Fields:
@@ -87,8 +87,8 @@ class TrackerPatternsDB:
try:
resp = http_get(url, timeout=3)
except RequestException as exc:
log.warning("TRACKER_PATTERNS: RequestException while fetching %s: %s", url, exc)
except HTTPError as exc:
log.warning("TRACKER_PATTERNS: HTTPError (%s) occured while fetching %s", url, exc)
continue
if resp.status_code != 200:

4
searx/data/useragents.json
View File

@@ -5,7 +5,7 @@
],
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
"versions": [
"154.0",
"153.0"
"153.0",
"152.0"
]
}

23618
searx/data/wikidata_properties.json
View File

File diff suppressed because it is too large Load Diff

10
searx/data/wikidata_units.json
View File

@@ -3474,6 +3474,11 @@
"symbol": "mm⁻²",
"to_si_factor": 1e-06
},
"Q136039973": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q1361854": {
"si_name": "Q11570",
"symbol": "dwt",
@@ -5249,11 +5254,6 @@
"symbol": "μA",
"to_si_factor": 1e-06
},
"Q31274648": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q3186734": {
"si_name": "Q3186734",
"symbol": "J/(m³ K)",

5
searx/enginelib/__init__.py
View File

@@ -305,7 +305,7 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
region: str = ""
"""For an engine, when there is ``region: ...`` in the YAML settings the engine
does support only this one region:
does support only this one region::
.. code:: yaml
@@ -317,9 +317,6 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
enable_http: bool
"""Enable HTTP (by default only HTTPS is enabled)."""
enable_http3: bool = False
"""Enables the use of HTTP/3 if available"""
shortcut: str
"""Code used to execute bang requests (``!foo``)"""

28
searx/engines/360search_videos.py
View File

@@ -6,8 +6,7 @@ from urllib.parse import urlencode
from datetime import datetime
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
from searx.utils import html_to_text
from searx.utils import html_to_text, get_embeded_stream_url
about = {
"website": "https://tv.360kan.com/",
@@ -30,12 +29,12 @@ def request(query, params):
return params
def response(resp) -> EngineResults:
def response(resp):
try:
data = resp.json()
except Exception as e:
raise SearxEngineAPIException(f"Invalid response: {e}") from e
res = EngineResults()
results = []
if "data" not in data or "result" not in data["data"]:
raise SearxEngineAPIException("Invalid response")
@@ -51,15 +50,16 @@ def response(resp) -> EngineResults:
except (ValueError, TypeError):
published_date = None
res.add(
res.types.LegacyResult(
url=entry["play_url"],
title=html_to_text(entry["title"]),
content=html_to_text(entry["description"]),
template='videos.html',
publishedDate=published_date,
thumbnail=entry["cover_img"],
)
results.append(
{
'url': entry["play_url"],
'title': html_to_text(entry["title"]),
'content': html_to_text(entry["description"]),
'template': 'videos.html',
'publishedDate': published_date,
'thumbnail': entry["cover_img"],
"iframe_src": get_embeded_stream_url(entry["play_url"]),
}
)
return res
return results

1
searx/engines/__builtins__.pyi
View File

@@ -26,7 +26,6 @@ categories: list[str]
disabled: bool
display_error_messages: bool
enable_http: bool
enable_http3: bool
engine_type: str
inactive: bool
max_page: int

3
searx/engines/__init__.py
View File

@@ -187,9 +187,8 @@ def set_loggers(engine: "Engine|types.ModuleType", engine_name: str):
def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]):
# pylint: disable=too-many-branches
# set / update engine attributes from engine_data
# set engine attributes from engine_data
kvargs: dict[str, t.Any]
engine.about = getattr(engine, "about", EngineAbout())
if isinstance(engine.about, EngineAbout):
kvargs = {**msgspec.to_builtins(engine.about), **engine_data.get("about", {})}
else:

10
searx/engines/adobe_stock.py
View File

@@ -35,7 +35,7 @@ Implementation
==============
"""
import typing as t
from datetime import datetime, timedelta
from urllib.parse import urlencode
@@ -85,7 +85,7 @@ Additional subcategories:
# Do we need support for "free_collection" and "include_stock_enterprise"?
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not categories:
raise ValueError("adobe_stock engine: categories is unset")
@@ -100,9 +100,9 @@ def setup(_: dict[str, t.Any]) -> bool | None:
raise ValueError("adobe_stock engine: adobe_content_types is unset")
if isinstance(adobe_content_types, list):
for content_type in adobe_content_types:
if content_type not in ADOBE_VALID_TYPES:
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % content_type)
for t in adobe_content_types:
if t not in ADOBE_VALID_TYPES:
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % t)
else:
raise ValueError(
"adobe_stock engine: adobe_content_types must be a list of strings not %s" % type(adobe_content_types)

8
searx/engines/baidu.py
View File

@@ -49,9 +49,6 @@ CACHE: EngineCache
def setup(engine_settings: dict[str, t.Any]) -> bool:
if baidu_category not in ('general', 'images', 'it'):
raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
return True
@@ -68,6 +65,11 @@ def get_image_cookies(headers: dict[str, str]) -> dict[str, str]:
return cookies
def init(_):
if baidu_category not in ('general', 'images', 'it'):
raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
def request(query, params):
page_num = params["pageno"]

33
searx/engines/bing.py
View File

@@ -40,7 +40,6 @@ about: dict[str, t.Any] = {
# engine dependent config
categories = ["general", "web"]
safesearch = True
enable_http3 = True
_safesearch_map: dict[int, str] = {
0: "off",
1: "moderate",
@@ -72,21 +71,43 @@ def get_locale_params(engine_region: str | None) -> dict[str, str] | None:
return {"mkt": engine_region}
def override_accept_language(params: "OnlineParams", engine_region: str | None) -> None:
"""Override the ``Accept-Language`` header.
The default header built by :py:class:`~searx.search.processors.online.OnlineProcessor`
appends ``en;q=0.3`` as a fallback language::
Accept-Language: de,de-DE;q=0.7,en;q=0.3
Bing seems to better select the results locale based on the
``Accept-Language`` value header.
This function is shared with :py:mod:`searx.engines.bing_images`,
:py:mod:`searx.engines.bing_news`, and :py:mod:`searx.engines.bing_videos`.
"""
if not engine_region or engine_region == "clear":
return
lang = engine_region.split("-")[0]
params["headers"]["Accept-Language"] = f"{engine_region},{lang};q=0.9"
def request(query: str, params: "OnlineParams"):
"""Assemble a Bing-Web request."""
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
override_accept_language(params, engine_region)
query_params: dict[str, str | int] = {
"q": query,
"adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
}
if engine_region and engine_region != "clear":
lang, _, cc = engine_region.partition("-")
query_params["setlang"] = lang
if cc and cc not in ("us", "cn", "ru"): # bing just sends junk for these
query_params["cc"] = cc
locale_params = get_locale_params(engine_region)
if locale_params:
query_params.update(locale_params)
params["url"] = f"{base_url}/search?{urlencode(query_params)}"

4
searx/engines/bing_images.py
View File

@@ -9,6 +9,7 @@ from lxml import html
from searx.engines.bing import ( # pylint: disable=unused-import
fetch_traits,
get_locale_params,
override_accept_language,
)
# about
@@ -24,7 +25,6 @@ about = {
# engine dependent config
categories = ["images", "web"]
paging = True
enable_http3 = True
safesearch = True
time_range_support = True
time_map = {
@@ -43,6 +43,8 @@ def request(query, params):
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
override_accept_language(params, engine_region)
# build URL query
# - example: https://www.bing.com/images/async?q=foo&async=1&first=1&count=35
query_params = {

8
searx/engines/bing_news.py
View File

@@ -12,7 +12,10 @@ from urllib.parse import urlencode
from lxml import html
from searx.enginelib.traits import EngineTraits
from searx.engines.bing import get_locale_params
from searx.engines.bing import (
get_locale_params,
override_accept_language,
)
from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
# about
@@ -30,7 +33,6 @@ categories = ["news"]
paging = True
"""If go through the pages and there are actually no new results for another
page, then bing returns the results from the last page again."""
enable_http3 = True
time_range_support = True
time_map = {
@@ -51,6 +53,8 @@ def request(query, params):
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
override_accept_language(params, engine_region)
# build URL query
# - example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
page = int(params.get("pageno", 1)) - 1

4
searx/engines/bing_videos.py
View File

@@ -9,6 +9,7 @@ from lxml import html
from searx.engines.bing import ( # pylint: disable=unused-import
fetch_traits,
get_locale_params,
override_accept_language,
)
from searx.engines.bing_images import time_map
from searx.utils import eval_xpath, eval_xpath_getindex
@@ -25,7 +26,6 @@ about = {
# engine dependent config
categories = ["videos", "web"]
paging = True
enable_http3 = True
safesearch = True
time_range_support = True
@@ -38,6 +38,8 @@ def request(query, params):
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
override_accept_language(params, engine_region)
# build URL query
# - example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
query_params = {

2
searx/engines/boardreader.py
View File

@@ -45,7 +45,7 @@ CACHE_SESSION_ID_KEY = "session_id_key"
KEYWORD_RE = re.compile(r"\[\/?Keyword\]")
def setup(engine_settings: dict[str, t.Any]) -> bool:
def init(engine_settings: dict[str, t.Any]) -> bool:
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_name=engine_settings["name"])
return True

30
searx/engines/brave.py
View File

@@ -135,7 +135,7 @@ from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
extract_text,
get_embedded_stream_url,
get_embeded_stream_url,
js_obj_str_to_json_str,
js_obj_str_to_python,
)
@@ -151,7 +151,6 @@ about = {
base_url = "https://search.brave.com/"
categories = []
enable_http3 = True
brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = "search"
"""Brave supports common web-search, videos, images, news, and goggles search.
@@ -248,13 +247,13 @@ def extract_json_data(text: str) -> dict[str, t.Any]:
# node_ids: [0, 19],
# data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# form: null,
# error: null
# });
text = text[text.index("<script") : text.index("</script")]
if not text:
raise ValueError("can't find JS/JSON data in the given text")
start = text.index("data: [{")
newline = text.index("\n", start)
end = text.rindex("}}]", start, newline)
js_obj_str = "{" + text[start:end] + "}}]}"
end = text.rindex("}}]")
js_obj_str = text[start:end]
js_obj_str = "{" + js_obj_str + "}}]}"
# js_obj_str = js_obj_str.replace("\xa0", "") # remove ASCII for &nbsp;
# js_obj_str = js_obj_str.replace(r"\u003C", "<").replace(r"\u003c", "<") # fix broken HTML tags in strings
json_str = js_obj_str_to_json_str(js_obj_str)
@@ -339,7 +338,7 @@ def _parse_search(resp: SXNG_Response) -> EngineResults:
if len(video_tag):
# In my tests a video tag in the WEB search was most often not a
# video, except the ones from youtube ..
iframe_src = get_embedded_stream_url(url)
iframe_src = get_embeded_stream_url(url)
if iframe_src:
item["iframe_src"] = iframe_src
item["template"] = "videos.html"
@@ -354,14 +353,14 @@ def _parse_news(resp: SXNG_Response) -> EngineResults:
res = EngineResults()
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, "//div[@data-type='news']"):
url = eval_xpath_getindex(result, ".//a/@href", 0, default=None)
for result in eval_xpath_list(dom, "//div[contains(@class, 'results')]//div[@data-type='news']"):
url = eval_xpath_getindex(result, ".//a[contains(@class, 'result-header')]/@href", 0, default=None)
if url is None:
continue
title = eval_xpath_list(result, ".//div[contains(@class, 'title')]")
content = eval_xpath_list(result, ".//div[contains(@class, 'description')]")
thumbnail = eval_xpath_getindex(result, ".//a[contains(@class, 'thumbnail')]//img/@src", 0, default="")
title = eval_xpath_list(result, ".//span[contains(@class, 'snippet-title')]")
content = eval_xpath_list(result, ".//p[contains(@class, 'desc')]")
thumbnail = eval_xpath_getindex(result, ".//div[contains(@class, 'image-wrapper')]//img/@src", 0, default="")
item = res.types.LegacyResult(
template="default.html",
@@ -407,6 +406,9 @@ def _parse_videos(json_resp: dict[str, t.Any]) -> EngineResults:
)
if result["thumbnail"] is not None:
item["thumbnail"] = result["thumbnail"]["src"]
iframe_src = get_embeded_stream_url(result["url"])
if iframe_src:
item["iframe_src"] = iframe_src
res.add(item)

9
searx/engines/braveapi.py
View File

@@ -40,7 +40,7 @@ if t.TYPE_CHECKING:
about = {
"website": "https://api.search.brave.com/",
"wikidata_id": None,
"official_api_documentation": "https://api-dashboard.search.brave.com/api-reference/web/search/get",
"official_api_documentation": "https://api-dashboard.search.brave.com/documentation",
"use_official_api": True,
"require_api_key": True,
"results": "JSON",
@@ -63,10 +63,8 @@ base_url = "https://api.search.brave.com/res/v1/web/search"
time_range_map = {"day": "past_day", "week": "past_week", "month": "past_month", "year": "past_year"}
"""Mapping of SearXNG time ranges to Brave API time ranges."""
max_page = 10
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
"""Initialize the engine."""
if not api_key:
raise SearxEngineAPIException("No API key provided")
@@ -77,7 +75,7 @@ def request(query: str, params: "OnlineParams") -> None:
search_args: dict[str, str | int | None] = {
"q": query,
"count": results_per_page,
"offset": params["pageno"] - 1,
"offset": (params["pageno"] - 1) * results_per_page,
"text_decorations": False,
}
@@ -91,7 +89,6 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = f"{base_url}?{urlencode(search_args)}"
params["headers"]["X-Subscription-Token"] = api_key
params["headers"]["Accept"] = "application/json"
def _extract_published_date(published_date_raw: str):

85
searx/engines/cara.py Normal file
View File

@@ -0,0 +1,85 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=invalid-name
"""Cara_ is a social media and portfolio-sharing platform for artists and art
enthusiasts.
With the widespread use of generative AI, Cara_ decided to build a place that
filters out gen AI images so that people searching for authentic creatives and
images can do so easily.
.. _Cara: https://cara.app/about
"""
from urllib.parse import urlencode
import typing as t
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://cara.app",
"official_api_documentation": None,
"use_official_api": False,
"require_api_key": False,
"results": "JSON",
}
base_url = "https://cara.app"
images_url = "https://images.cara.app"
categories = ["images"]
paging = True
results_per_page = 24
# if using HTTP2, we get blocked immediately
enable_http2 = False
def request(query: str, params: "OnlineParams") -> None:
args = {
"q": query,
"sortBy": "Top",
"take": results_per_page,
"skip": (params["pageno"] - 1) * results_per_page,
}
params["url"] = f"{base_url}/api/search/portfolio-posts?{urlencode(args)}"
def response(resp: "SXNG_Response"):
res = EngineResults()
json_data: list[dict[str, t.Any]] = resp.json()
for result in json_data:
thumbnail, img = None, None
i: dict[str, str]
for i in result["images"]:
if thumbnail is None or i["isCoverImg"]:
thumbnail = i
if img is None or not i["isCoverImg"]:
img = i
if not thumbnail or not img:
continue
res.add(
res.types.LegacyResult(
{
"template": "images.html",
"url": f"{base_url}/post/{result['id']}",
"thumbnail_src": f"{images_url}/{thumbnail['src']}?height=256",
"img_src": f"{images_url}/{img['src']}",
"title": result["title"],
"content": result["content"],
"author": result["name"],
}
)
)
return res

10
searx/engines/chatnoir.py
View File

@@ -41,7 +41,7 @@ search_index = "cw22"
<https://www.chatnoir.eu/docs/api-general>`_ for a full list."""
def _obtain_api_key() -> tuple[str, str]:
def _obtain_api_key() -> tuple[str, str, str]:
home_resp = get(base_url)
if not home_resp.ok:
raise SearxEngineAPIException("failed to obtain api key")
@@ -58,9 +58,10 @@ def _obtain_api_key() -> tuple[str, str]:
)
if not token_resp.ok:
raise SearxEngineAPIException("failed to obtain api key")
session_id = token_resp.cookies["sessionid"]
scraped_api_key = token_resp.json()["token"]["token"]
return csrf_token, scraped_api_key
return csrf_token, session_id, scraped_api_key
def request(query: str, params: "OnlineParams"):
@@ -72,7 +73,7 @@ def request(query: str, params: "OnlineParams"):
params["headers"].update(headers)
else:
csrf_token, scraped_api_key = _obtain_api_key()
csrf_token, session_id, scraped_api_key = _obtain_api_key()
headers = {
"Authorization": f"Bearer {scraped_api_key}",
@@ -80,11 +81,10 @@ def request(query: str, params: "OnlineParams"):
}
params["headers"].update(headers)
params["cookies"] = {"csrftoken": csrf_token}
params["cookies"] = {"csrftoken": session_id, "sessionid": session_id}
params["url"] = f"{base_url}/api/v1/_search"
params["method"] = "POST"
params["impersonate"] = "none"
json_data = {
"query": query,

2
searx/engines/chinaso.py
View File

@@ -78,7 +78,7 @@ time_range_dict = {'day': '24h', 'week': '1w', 'month': '1m', 'year': '1y'}
base_url = "https://www.chinaso.com"
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if chinaso_news_source not in t.get_args(ChinasoNewsSourceType):
raise ValueError(f"Unsupported news source: {chinaso_news_source}")

3
searx/engines/command.py
View File

@@ -74,7 +74,6 @@ Implementations
===============
"""
import typing as t
import re
from os.path import expanduser, isabs, realpath, commonprefix
from shlex import split as shlex_split
@@ -101,7 +100,7 @@ _command_logger = logger.getChild('command')
_compiled_parse_regex = {}
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
check_parsing_options(engine_settings)
if 'command' not in engine_settings:

3
searx/engines/core.py
View File

@@ -141,13 +141,12 @@ def response(resp: "SXNG_Response") -> EngineResults:
if name:
authors.add(name)
tag = result.get("fieldOfStudy")
res.add(
res.types.Paper(
title=result.get("title"),
url=url,
content=result.get("fullText", "") or "",
tags=[tag] if tag else [],
tags=result.get("fieldOfStudy", []),
publishedDate=published_date,
type=result.get("documentType", "") or "",
authors=authors,

26
searx/engines/dogpile.py
View File

@@ -8,9 +8,6 @@ import typing as t
from datetime import datetime, timezone
import html
from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineAPIException
from searx.network import post
from searx.utils import format_duration, html_to_text, humanize_number
from searx.result_types import EngineResults
@@ -38,36 +35,15 @@ dogpile_categ = "search"
base_url = "https://www.dogpile.com"
safe_search_map = {0: "none", 1: "moderate", 2: "heavy"}
CACHE: EngineCache
"""Cache for the API token from dogpile"""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if dogpile_categ not in ("search", "images", "videos", "news"):
raise ValueError("invalid search type: %s" % dogpile_categ)
global CACHE # pylint: disable=global-statement
CACHE = EngineCache("dogpile") # one token for images/videos/news
return True
def _obtain_token() -> str:
token = CACHE.get("token")
if token:
return token
resp = post(f"{base_url}/api/token/refresh", headers={"Origin": base_url}, cookies={"dp_api_token": "1"})
if not resp.ok:
raise SearxEngineAPIException("failed to obtain dogpile token")
token = resp.json()["token"]
CACHE.set("token", token, expire=240) # 300s ttl
return token
def request(query: str, params: "OnlineParams"):
params["url"] = f"{base_url}/api/{dogpile_categ}"
params["headers"]["Origin"] = base_url
params["cookies"]["dp_api_token"] = "1"
if dogpile_categ != "search": # web doesnt need token
params["headers"]["x-dogpile-token"] = _obtain_token()
params["method"] = "POST"
params["json"] = {"q": query, "qadf": safe_search_map[params["safesearch"]], "page": params["pageno"]}

71
searx/engines/duckduckgo_extra.py
View File

@@ -10,8 +10,7 @@ from datetime import datetime
from urllib.parse import urlencode
from urllib.parse import quote_plus
from searx.result_types import EngineResults, MainResult, LegacyResult, Image
from searx.utils import html_to_text, gen_useragent, extr
from searx.utils import get_embeded_stream_url, html_to_text, gen_useragent, extr
from searx.network import get # see https://github.com/searxng/searxng/issues/762
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
@@ -48,7 +47,7 @@ _HTTP_User_Agent: str = gen_useragent()
send_accept_language_header = False
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings: dict[str, t.Any]):
if engine_settings["ddg_category"] not in ["images", "videos", "news"]:
raise ValueError(f"Unsupported DuckDuckGo category: {engine_settings['ddg_category']}")
@@ -98,7 +97,6 @@ def request(query: str, params: "OnlineParams") -> None:
# The vqd value is generated from the query and the UA header. To be able to
# reuse the vqd value, the UA header must be static.
headers["User-Agent"] = _HTTP_User_Agent
params["impersonate"] = "none"
vqd = get_vqd(query=query, params=params) or fetch_vqd(query=query, params=params)
headers["Accept"] = "*/*"
@@ -150,51 +148,54 @@ def request(query: str, params: "OnlineParams") -> None:
def _image_result(result):
return Image(
url=result['url'],
title=result['title'],
content='',
thumbnail_src=result['thumbnail'],
img_src=result['image'],
resolution='%s x %s' % (result['width'], result['height']),
source=result['source'],
)
return {
'template': 'images.html',
'url': result['url'],
'title': result['title'],
'content': '',
'thumbnail_src': result['thumbnail'],
'img_src': result['image'],
'resolution': '%s x %s' % (result['width'], result['height']),
'source': result['source'],
}
def _video_result(result):
return LegacyResult(
template='videos.html',
url=result['content'],
title=result['title'],
content=result['description'],
thumbnail=result['images'].get('small') or result['images'].get('medium'),
source=result['provider'],
length=result['duration'],
metadata=result.get('uploader'),
)
return {
'template': 'videos.html',
'url': result['content'],
'title': result['title'],
'content': result['description'],
'thumbnail': result['images'].get('small') or result['images'].get('medium'),
'iframe_src': get_embeded_stream_url(result['content']),
'source': result['provider'],
'length': result['duration'],
'metadata': result.get('uploader'),
}
def _news_result(result):
return MainResult(
url=result['url'],
title=result['title'],
content=html_to_text(result['excerpt']),
publishedDate=datetime.fromtimestamp(result['date']),
)
return {
'url': result['url'],
'title': result['title'],
'content': html_to_text(result['excerpt']),
'source': result['source'],
'publishedDate': datetime.fromtimestamp(result['date']),
}
def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
def response(resp):
results = []
res_json = resp.json()
for result in res_json['results']:
if ddg_category == 'images':
res.add(_image_result(result))
results.append(_image_result(result))
elif ddg_category == 'videos':
res.add(_video_result(result))
results.append(_video_result(result))
elif ddg_category == 'news':
res.add(_news_result(result))
results.append(_news_result(result))
else:
raise ValueError(f"Invalid duckduckgo category: {ddg_category}")
return res
return results

14
searx/engines/duckduckgo_weather.py
View File

@@ -109,19 +109,7 @@ def response(resp: SXNG_Response):
json_data = loads(resp.text[resp.text.find('\n') + 1 : resp.text.rfind('\n') - 2])
location = json_data.get("location")
if not location:
return res
metadata = json_data.get("weatherAlerts", {}).get("metadata", {})
geoloc = weather.GeoLocation(
name=location,
latitude=metadata.get("latitude"),
longitude=metadata.get("longitude"),
elevation=0,
country_code=metadata.get("language").split("-")[-1],
timezone=json_data.get("location"),
)
geoloc = weather.GeoLocation.by_query(resp.search_params["query"])
weather_answer = EngineResults.types.WeatherAnswer(
current=_weather_data(geoloc, json_data["currentWeather"]),

96
searx/engines/duckduckgo_web.py
View File

@@ -14,12 +14,11 @@ can't build it ourselves and must scrape it from the HTML pages.
"""
import typing as t
import re
from urllib.parse import quote_plus, urljoin
from urllib.parse import quote_plus
from lxml import html
from searx.utils import html_to_text, extract_text, eval_xpath
from searx.utils import html_to_text, gen_useragent, extract_text, eval_xpath
from searx.result_types import EngineResults
from searx.enginelib import EngineCache
from searx.network import get
@@ -39,6 +38,7 @@ about = {
# engine dependent config
categories = ["general"]
paging = True
_HTTP_User_Agent: str = gen_useragent()
base_url = "https://duckduckgo.com"
@@ -73,8 +73,6 @@ def _fetch_first_page_link(
resp = get(
url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
headers=headers,
impersonate="firefox",
default_headers=False,
timeout=2,
)
@@ -98,43 +96,6 @@ def _cache_key(query: str, pageno: int) -> str:
return f"nextpage_url|{query}|{pageno}"
def _solve_jsa(resp: "SXNG_Response") -> "SXNG_Response":
"""Duckduckgo sometimes issues a challenge instead of json."""
# length that a real browser would report for where the broken snippet is
html_len = {
"<p><div></p><p></div": 32,
"<li><div></li><li></div": 29,
"<div><div></div><div></div": 33,
"<br><div></br><br></div": 23,
}
js = resp.text or ""
jsa_match = re.search(r"let jsa = (\d+);.*?DDG\.deep\.initialize\('([^']+)'", js, re.S)
if not jsa_match:
return resp
js_functions = dict(re.findall(r"let (\w+) = function\(num\) \{([^}]*)\};", js))
jsa = int(jsa_match.group(1))
try:
for name in re.findall(r"jsa = (\w+)\(jsa\);", js):
body = js_functions[name]
mul = re.search(r"num \* (\d+)", body)
jsa = jsa * int(mul.group(1)) if mul else jsa + html_len[re.search(r"`([^`]+)`", body).group(1)]
except (KeyError, AttributeError):
return resp
params = resp.search_params
follow = get(
urljoin("https://links.duckduckgo.com", jsa_match.group(2) + str(jsa)),
headers=params["headers"],
impersonate="firefox",
default_headers=False,
)
follow.search_params = params
return follow
def request(query: str, params: "OnlineParams") -> None:
if len(query) >= 500:
@@ -142,15 +103,25 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = None
return
# firefox TLS only
params["impersonate"] = "firefox"
params["default_headers"] = False
headers = params["headers"]
# The vqd value is generated from the query and the UA header. To be able
# to reuse the vqd value, the UA header must be static.
headers["User-Agent"] = _HTTP_User_Agent
headers["Accept"] = "*/*"
headers["Referer"] = f"{base_url}/"
headers["Host"] = "duckduckgo.com"
# Sec-Fetch headers are required to not get blocked when sending a Firefox user agent
headers["Sec-Fetch-Dest"] = "script"
headers["Sec-Fetch-Mode"] = "no-cors"
headers["Sec-Fetch-Site"] = "same-site"
api_url = ""
if params["pageno"] > 1:
api_url = CACHE.get(_cache_key(query, params["pageno"]))
else:
api_url = _fetch_first_page_link(query, params["headers"])
api_url = _fetch_first_page_link(query, headers)
if not api_url:
params["url"] = None
@@ -158,27 +129,14 @@ def request(query: str, params: "OnlineParams") -> None:
params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
# loads as a script
headers = params["headers"]
headers["Accept"] = "*/*"
headers["Sec-Fetch-Dest"] = "script"
headers["Sec-Fetch-Mode"] = "no-cors"
headers["Sec-Fetch-Site"] = "same-site"
headers["Referer"] = f"{base_url}/"
# TODO: support safesearch, timerange and engine traits # pylint:disable=fixme
def response(resp: "SXNG_Response"):
res = EngineResults()
res_json = resp.json()
# check if ddg returns a challenge
# e.g. 'site:github.com searxng'
if "let jsa =" in (resp.text or ""):
resp = _solve_jsa(resp)
results = resp.json()["results"]
for result in results:
for result in res_json["results"]:
if "u" not in result:
continue
@@ -186,13 +144,13 @@ def response(resp: "SXNG_Response"):
res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
)
if results:
next_page_path = results[-1].get("n")
if next_page_path:
CACHE.set(
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
base_url + next_page_path,
expire=60 * 60,
)
# link to next page
next_page_path = res_json["results"][-1].get("n")
if next_page_path:
CACHE.set(
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
base_url + next_page_path,
expire=60 * 60,
)
return res

3
searx/engines/elasticsearch.py
View File

@@ -41,7 +41,6 @@ authentication configured to read from ``my-index`` index.
"""
import typing as t
from json import loads, dumps
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
@@ -69,7 +68,7 @@ show_metadata = False
page_size = 10
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
if 'query_type' in engine_settings and engine_settings['query_type'] not in _available_query_types:
raise ValueError('unsupported query type', engine_settings['query_type'])

150
searx/engines/europepmc.py
View File

@@ -1,150 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""`Europe PMC`_ provides comprehensive access to life sciences literature from
trusted sources. With Europe PMC you can search and read millions of
publications, preprints and other documents enriched with links to supporting
data, reviews, protocols, and other relevant resources.
.. _Europe PMC: https://europepmc.org/
Configuration
=============
.. code:: yaml
- name: europepmc
engine: europepmc
shortcut: epmc
Implementations
===============
"""
import typing as t
from datetime import datetime
from urllib.parse import urlencode
from dateutil.parser import isoparse
from searx.enginelib import EngineCache
from searx.result_types import EngineResults
from searx.utils import html_to_text
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://europepmc.org/",
"wikidata_id": "Q5412157",
"official_api_documentation": "https://europepmc.org/RestfulWebService",
"use_official_api": True,
"require_api_key": False,
"results": "JSON",
}
categories = ["science", "scientific publications"]
paging = True
# engine dependent config
search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
article_url = "https://europepmc.org/article/"
page_size = 20
CACHE: EngineCache
"""Cache for storing the pagination cursor."""
def setup(engine_settings: dict[str, t.Any]):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
def _cache_key(query: str, page: int) -> str:
return f"{query}|{page}"
def request(query: str, params: "OnlineParams") -> None:
args = {
"query": query,
"format": "json",
"resultType": "core",
"pageSize": page_size,
}
if params["pageno"] > 1:
if cursor := CACHE.get(_cache_key(query, params["pageno"])):
args["cursorMark"] = cursor
else:
# no cached cursor for that page
params["url"] = None
return
params["url"] = f"{search_url}?{urlencode(args)}"
def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
json_resp = resp.json()
# store pagination cursor for loading next pages in cache
if next_cursor := json_resp.get("nextCursorMark"):
next_page = resp.search_params["pageno"] + 1
query = resp.search_params["query"]
CACHE.set(_cache_key(query, next_page), next_cursor)
all_results = json_resp.get("resultList", {}).get("result", [])
for item in all_results:
source = item.get("source", "")
identifier = item.get("id", "")
url = f"{article_url}{source}/{identifier}" if source and identifier else ""
journal_info: dict[str, t.Any] = item.get("journalInfo", {})
journal: dict[str, t.Any] = journal_info.get("journal", {})
res.add(
res.types.Paper(
url=url,
title=html_to_text(item.get("title", "")),
content=html_to_text(item.get("abstractText", "")),
journal=journal.get("title", ""),
issn=[journal.get("issn", "")],
authors=_get_authors(item),
doi=item.get("doi", ""),
publishedDate=_get_published_date(item),
type=", ".join((item.get("pubTypeList", {})).get("pubType", [])),
pdf_url=_get_pdf_url(item),
html_url=url,
)
)
return res
def _get_authors(item: dict[str, t.Any]) -> list:
"""Extract the list of authors from the item."""
if authors := item.get("authorString", None):
authors = [author.strip().rstrip(".") for author in authors.split(",") if author.strip()]
else:
authors = []
return authors
def _get_pdf_url(item: dict[str, t.Any]) -> str:
"""Extract the PDF URL in case it is open access."""
for url_info in (item.get("fullTextUrlList", {})).get("fullTextUrl", []):
if url_info.get("documentStyle") == "pdf" and url_info.get("availabilityCode") == "OA":
return url_info.get("url", "")
return ""
def _get_published_date(item: dict[str, t.Any]) -> datetime | None:
"""Extract the published date from the item and convert it to a datetime object."""
if unformatted_date := item.get("firstPublicationDate"):
return isoparse(unformatted_date)
return None

2
searx/engines/exaapi.py
View File

@@ -78,7 +78,7 @@ content_max_characters: int = 500
"""Maximum characters for the requested content."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not api_key:
raise SearxEngineAPIException("No API key provided")
if not 1 <= results_per_page <= 100:

2
searx/engines/fireball.py
View File

@@ -47,7 +47,7 @@ CACHE_VALID_DURATION = 30 * 24 * 3600 # one month, same as website
"""Duration how long settings cookies are valid."""
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings: dict[str, t.Any]):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])

3
searx/engines/gitea.py
View File

@@ -38,7 +38,6 @@ Implementation
"""
import typing as t
from urllib.parse import urlencode
from dateutil import parser
@@ -78,7 +77,7 @@ page_size: int = 10
"""Maximum number of results per page (default 10)."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not base_url:
raise ValueError('gitea engine: base_url is unset')

264
searx/engines/google.py
View File

@@ -9,15 +9,12 @@ engines:
- :ref:`google scholar engine`
- :ref:`google autocomplete`
This implementation uses Nokia user agents to request an XML layout from Google.
The normal web version requires executing JavaScript to load the results and
therefore is currently not used here. See `Google discussion`_ for more
information on that topic.
.. _Google discussion: https://github.com/searxng/searxng/issues/6359
"""
import random
import re
import string
import time
import typing as t
from urllib.parse import unquote, urlencode
@@ -47,16 +44,16 @@ about = {
"official_api_documentation": "https://developers.google.com/custom-search/",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["general", "web"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
@@ -67,23 +64,38 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
# Filter results. 0: None, 1: Moderate, 2: Strict
filter_mapping = {0: "off", 1: "medium", 2: "high"}
# https://github.com/searxng/searxng/issues/6359
nokia_useragents = (
"Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
)
# specific xpath variables
# ------------------------
# Suggestions are links placed in a *card-section*, we extract only the text
# from the links not the links itself.
suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
_arcid_range = string.ascii_letters + string.digits + "_-"
_arcid_random: tuple[str, int] | None = None
def ui_async(start: int) -> str:
"""Format of the response from UI's async request.
- ``arc_id:<...>,use_ac:true,_fmt:prog``
The arc_id is random generated every hour.
"""
global _arcid_random # pylint: disable=global-statement
use_ac = "use_ac:true"
# _fmt:html returns a HTTP 500 when user search for celebrities like
# '!google natasha allegri' or '!google chris evans'
_fmt = "_fmt:prog"
# create a new random arc_id every hour
if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
_arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
return ",".join([arc_id, use_ac, _fmt])
def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
@@ -115,11 +127,19 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
A instance of :py:obj:`babel.core.Locale` build from the
``searxng_locale`` value.
subdomain:
Google subdomain :py:obj:`google_domains` that fits to the country
code.
params:
Py-Dictionary with additional request arguments (can be passed to
:py:func:`urllib.parse.urlencode`).
- ``hl`` parameter: specifies the interface language of user interface.
- ``lr`` parameter: restricts search results to documents written in
a particular language.
- ``cr`` parameter: restricts search results to documents
originating in a particular country.
- ``ie`` parameter: sets the character encoding scheme that should
be used to interpret the query string ('utf8').
- ``oe`` parameter: sets the character encoding scheme that should
@@ -136,6 +156,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val: dict[str, t.Any] = {
"language": None,
"country": None,
"subdomain": None,
"params": {},
"headers": {},
"cookies": {},
@@ -148,7 +169,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
except babel.core.UnknownLocaleError:
locale = None
eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
lang_code = eng_lang.split("_")[-1] # lang_zh-TW --> zh-TW / lang_en --> en
country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
@@ -163,6 +184,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val["language"] = eng_lang
ret_val["country"] = country
ret_val["locale"] = locale
ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
# hl parameter:
# The hl parameter specifies the interface language (host language) of
@@ -201,11 +223,9 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
# specify a region (country) only if a region is given in the selected
# locale --> https://github.com/searxng/searxng/issues/2672
if country is not None:
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
# gl parameter: (mandatory by Google News)
# The gl parameter value is a two-letter country code. For WebSearch
@@ -280,78 +300,88 @@ def detect_google_sorry(resp: "SXNG_Response"):
raise SearxEngineCaptchaException()
def unwrap_google_url(raw_url: str) -> str:
# remove redirector from url
if raw_url.startswith("/url?q="):
return unquote(raw_url[7:].split("&sa=U")[0])
return raw_url
def wml_dom(resp: "SXNG_Response"):
detect_google_sorry(resp)
text = resp.text
if text.lstrip().startswith("<?xml"):
text = text.split("?>", 1)[-1]
return html.fromstring(text)
def google_request(
query: str,
params: "OnlineParams",
extra_args: dict[str, t.Any] | None = None,
*,
eng_traits: EngineTraits | None = None,
use_time_range: bool = True,
use_safesearch: bool = True,
safesearch_map: dict[int, str] | None = None,
use_locales: bool = True,
) -> None:
google_info = get_google_info(params, eng_traits or traits)
if not use_locales:
google_info["params"].pop("lr")
google_info["params"].pop("cr")
start = (params["pageno"] - 1) * 10
args: dict[str, t.Any] = {
"q": query,
"sca_esv": "1",
**google_info["params"],
**(extra_args or {}),
}
if start:
args["start"] = start
if use_time_range and params["time_range"] in time_range_dict:
args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
if use_safesearch and params["safesearch"]:
args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
params["headers"]["User-Agent"] = random.choice(nokia_useragents)
params["impersonate"] = "chrome99_android"
def request(query: str, params: "OnlineParams") -> None:
google_request(query, params)
"""Google search request"""
# pylint: disable=line-too-long
start = (params["pageno"] - 1) * 10
google_info = get_google_info(params, traits)
# https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search"
+ "?"
+ urlencode(
{
"q": query,
**google_info["params"],
"filter": "0",
"start": start,
# 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
# 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
# 'cs' : 1,
# 'sa': 'N',
# 'yv': 3,
# 'prmd': 'vin',
# 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
# 'sa': 'N',
# 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
# formally known as use_mobile_ui
# "asearch": "arc",
# "async": str_async,
}
)
)
if params["time_range"] in time_range_dict:
query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
if params["safesearch"]:
query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
# regex match to get image map that is found inside the returned javascript:
# (function(){var s='...';var i=['...'] ...}
RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
def parse_url_images(text: str):
data_image_map = {}
for image_url, img_id in RE_DATA_IMAGE.findall(text):
data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def response(resp: "SXNG_Response"):
"""Get response from google's search request"""
# pylint: disable=too-many-branches, too-many-statements
detect_google_sorry(resp)
data_image_map = parse_url_images(resp.text)
results = EngineResults()
dom = wml_dom(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
# parse results
for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
# pylint: disable=too-many-nested-blocks
try:
title_tag = eval_xpath_getindex(
result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
)
title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
if title_tag is None:
# this not one of the common google results *section*
logger.debug("ignoring item from the result_xpath list: missing title")
continue
title = extract_text(title_tag)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
raw_url = result.get("href")
if raw_url is None:
logger.debug(
'ignoring item from the result_xpath list: missing url of title "%s"',
@@ -359,19 +389,30 @@ def response(resp: "SXNG_Response") -> EngineResults:
)
continue
url = unwrap_google_url(raw_url)
content = extract_text(
eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
)
thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
results.add(
results.types.MainResult(
url=url,
title=title or "",
content=content or "",
thumbnail=thumbnail or "",
)
)
if raw_url.startswith('/url?q='):
url = unquote(raw_url[7:].split("&sa=U")[0]) # remove the google redirector
else:
url = raw_url
content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
for item in content_nodes:
for script in item.xpath(".//script"):
script.getparent().remove(script)
content = extract_text(content_nodes[0])
# Images that are NOT the favicon
xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
thumbnail = None
if xpath_image is not None:
thumbnail = xpath_image.get("src")
if thumbnail.startswith("data:image"):
img_id = xpath_image.get("id")
if img_id:
thumbnail = data_image_map.get(img_id)
results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
except Exception as e: # pylint: disable=broad-except
logger.error(e, exc_info=True)
@@ -379,8 +420,10 @@ def response(resp: "SXNG_Response") -> EngineResults:
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
# append suggestion
results.append({"suggestion": extract_text(suggestion)})
# return results
return results
@@ -413,12 +456,14 @@ skip_countries = [
]
def fetch_traits(engine_traits: EngineTraits):
def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
"""Fetch languages from Google."""
# pylint: disable=import-outside-toplevel, too-many-branches
from searx.network import get # see https://github.com/searxng/searxng/issues/762
engine_traits.custom["supported_domains"] = {}
resp = get("https://www.google.com/preferences", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google preferences is not OK.")
@@ -469,3 +514,22 @@ def fetch_traits(engine_traits: EngineTraits):
# alias regions
engine_traits.regions["zh-CN"] = "HK"
# supported domains
if add_domains:
resp = get("https://www.google.com/supported_domains", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google supported domains is not OK.")
for domain in resp.text.split():
domain = domain.strip()
if not domain or domain in [
".google.com",
]:
continue
region = domain.split(".")[-1].upper()
engine_traits.custom["supported_domains"][region] = "www" + domain
if region == "HK":
# There is no google.cn, we use .com.hk for zh-CN
engine_traits.custom["supported_domains"]["CN"] = "www" + domain

8
searx/engines/google_cse.py
View File

@@ -30,7 +30,6 @@ about = {
categories = ["general", "web"]
paging = True
enable_http3 = True
max_page = 5
page_size = 20
time_range_support = True
@@ -96,11 +95,12 @@ def request(query: str, params: "OnlineParams") -> None:
token = _cse_token()
google_info = get_google_info(params, traits)
info: dict[str, str] = google_info["params"]
args = {
"rsz": "filtered_cse",
"num": str(page_size),
"hl": google_info["params"]["hl"],
"hl": info["hl"],
"cselibv": token["cselibv"],
"cx": CX,
"q": query,
@@ -114,6 +114,10 @@ def request(query: str, params: "OnlineParams") -> None:
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
args["sort"] = f"date:r:{start_date}:{end_date}"
if info.get("lr"):
args["lr"] = info["lr"]
if info.get("cr"):
args["cr"] = info["cr"]
if google_info["country"] not in (None, "ZZ"):
args["gl"] = google_info["country"]
if token["exp"]:

140
searx/engines/google_images.py
View File

@@ -1,76 +1,122 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Images: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Images engine using the internal
Google API used by the Google Go Android app.
import typing as t
from urllib.parse import parse_qs, unquote, urlparse
This internal API offer results in
- JSON (``_fmt:json``)
- Protobuf_ (``_fmt:pb``)
- Protobuf_ compressed? (``_fmt:pc``)
- HTML (``_fmt:html``)
- Protobuf_ encoded in JSON (``_fmt:jspb``).
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
"""
from urllib.parse import urlencode
from json import loads
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, wml_dom
from searx.result_types import EngineResults
from searx.utils import eval_xpath_list
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import (
get_google_info,
time_range_dict,
detect_google_sorry,
)
# about
about = {
"website": "https://images.google.com",
"wikidata_id": "Q521550",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://images.google.com',
"wikidata_id": 'Q521550',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'JSON',
}
# engine dependent config
categories = ["images", "web"]
categories = ['images', 'web']
paging = True
enable_http3 = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
safesearch = True
filter_mapping = {0: "images", 1: "active", 2: "active"}
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "isch"},
eng_traits=traits,
safesearch_map=filter_mapping,
use_locales=False,
def request(query, params):
"""Google-Image search request"""
google_info = get_google_info(params, traits)
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ '?'
+ urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
# don't urlencode this because wildly different AND bad results
# pagination uses Zero-based numbering
+ f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if params['safesearch']:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
# this ua will allow getting ~50 results instead of 10. #1641
params['headers']['User-Agent'] = (
'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
)
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
dom = wml_dom(resp)
return params
for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
qs = parse_qs(urlparse(link.get("href", "")).query)
img_src = qs.get("imgurl", [""])[0]
url = qs.get("imgrefurl", [""])[0]
if not img_src or not url:
continue
width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
tbnid = qs.get("tbnid", [""])[0]
results.add(
results.types.Image(
url=url,
title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
img_src=img_src,
thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
resolution=f"{width} x {height}" if width and height else "",
)
)
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
json_start = resp.text.find('{"ischj":')
json_data = loads(resp.text[json_start:])
for item in json_data["ischj"].get("metadata", []):
result_item = {
'url': item["result"]["referrer_url"],
'title': item["result"]["page_title"],
'content': item["text_in_grid"]["snippet"],
'source': item["result"]["site_title"],
'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
'img_src': item["original_image"]["url"],
'thumbnail_src': item["thumbnail"]["url"],
'template': 'images.html',
}
author = item["result"].get('iptc', {}).get('creator')
if author:
result_item['author'] = ', '.join(author)
copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
if copyright_notice:
result_item['source'] += ' | ' + copyright_notice
freshness_date = item["result"].get("freshness_date")
if freshness_date:
result_item['source'] += ' | ' + freshness_date
file_size = item.get('gsa', {}).get('file_size')
if file_size:
result_item['source'] += ' (%s)' % file_size
results.append(result_item)
return results

329
searx/engines/google_news.py
View File

@@ -1,91 +1,324 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google News: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google News engine.
Google News has a different region handling compared to Google WEB.
- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
- the hl_ argument has to be set correctly (and different to Google WEB)
- the gl_ argument is mandatory
If one of this argument is not set correctly, the request is redirected to
CONSENT dialog::
https://consent.google.com/m?continue=
The google news API ignores some parameters from the common :ref:`google API`:
- num_ : the number of search results is ignored / there is no paging all
results for a query term are in the first response.
- save_ : is ignored / Google-News results are always *SafeSearch*
.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
"""
import typing as t
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
import json
import base64
from urllib.parse import urlencode
from lxml import html
import babel
from searx import locales
from searx.utils import (
eval_xpath_getindex,
eval_xpath,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
)
from searx.engines.google import fetch_traits as _fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
detect_google_sorry,
)
from searx.enginelib.traits import EngineTraits
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
# about
about = {
"website": "https://www.google.com",
"website": "https://news.google.com",
"wikidata_id": "Q12020",
"official_api_documentation": "https://developers.google.com/custom-search",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["news"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
paging = False
time_range_support = False
language_support = True
safesearch = False
# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
# False here.
#
# safesearch : results are identical for safesearch=0 and safesearch=2
safesearch = True
base_url: str = "https://news.google.com"
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "nws"},
eng_traits=traits,
use_time_range=False,
use_safesearch=False,
use_locales=False,
"""Google-News search request"""
sxng_locale = params.get("searxng_locale", "en-US")
ceid: str = locales.get_engine_locale(
sxng_locale, traits.custom["ceid"], default="US:en"
) # pyright: ignore[reportAssignmentType]
google_info = get_google_info(params, traits)
google_info["subdomain"] = "news.google.com" # google news has only one domain
ceid_region, ceid_lang = ceid.split(":")
ceid_lang, ceid_suffix = (
ceid_lang.split(":")
+ [
"",
]
)[:2]
google_info["params"]["hl"] = ceid_lang
if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
if ceid_region.lower() == ceid_lang:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
elif ceid_region.lower() != ceid_lang:
if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
google_info["params"]["hl"] = ceid_lang
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
google_info["params"]["gl"] = ceid_region
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search?"
+ urlencode(
{"q": query, **google_info["params"]},
)
# ceid includes a ':' character which must not be urlencoded
+ ("&ceid=%s" % ceid)
)
def _span_text(link, css_class: str):
return extract_text(
eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
allow_none=True,
)
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
seen = set()
for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
href = link.get("href")
if not href:
"""Get response from google's search request"""
res = EngineResults()
detect_google_sorry(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
if not url:
continue
if url.startswith("./"):
url = base_url + url[1:]
# The real URL is often encoded in the "jslog" attribute
jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
# Try to extract the real URL from jslog
real_url: str | None = None
if jslog:
# jslog format is usually: "95014; 5:<base64>; track:click,vis". We
# want the second part (index 1) after splitting by ";"
parts: list[str] = jslog.split(";")
if len(parts) > 1:
b64_data: str = parts[1].split(":")[-1].strip()
# Pad base64 if necessary
b64_data += "=" * (-len(b64_data) % 4)
decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
# The URL is typically the last element in the decoded array
if (
isinstance(decoded_data, list)
and isinstance(decoded_data[-1], str)
and decoded_data[-1].startswith("http")
):
real_url = decoded_data[-1]
if real_url:
url = real_url
else:
logger.error(f"no real-url found: {url}")
continue
url = unwrap_google_url(href)
if url in seen or "google.com/search" in url:
continue
title = extract_text(eval_xpath(result, "./h4")) or ""
title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
if not title:
continue
# The pub_date is mostly a string like 'yesterday', not a real timezone
# date or time. Therefore we can't use publishedDate and place the
# *pub* sting into the content.
source = _span_text(link, "dXDvrc")
pub_date = _span_text(link, "YVIcad")
thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
pub_date = extract_text(eval_xpath(result, ".//time"))
pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
content = " / ".join([x for x in [pub_origin, pub_date] if x])
seen.add(url)
results.add(
results.types.MainResult(
thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
if thumbnail and thumbnail.startswith("/"):
thumbnail = base_url + thumbnail
res.add(
res.types.MainResult(
url=url,
title=title,
content=" / ".join(x for x in [source, pub_date] if x),
thumbnail=thumbnail or "",
content=content,
thumbnail=thumbnail,
)
)
return results
return res
ceid_list = [
"AE:ar",
"AR:es-419",
"AT:de",
"AU:en",
"BD:bn",
"BE:fr",
"BE:nl",
"BG:bg",
"BR:pt-419",
"BW:en",
"CA:en",
"CA:fr",
"CH:de",
"CH:fr",
"CL:es-419",
"CN:zh-Hans",
"CO:es-419",
"CU:es-419",
"CZ:cs",
"DE:de",
"EE:et",
"EG:ar",
"ES:ca",
"ES:es",
"ET:en",
"FI:fi",
"FR:fr",
"GB:en",
"GH:en",
"GR:el",
"HK:zh-Hant",
"HU:hu",
"ID:en",
"ID:id",
"IE:en",
"IL:en",
"IL:he",
"IN:bn",
"IN:en",
"IN:gu",
"IN:hi",
"IN:ml",
"IN:mr",
"IN:pa",
"IN:ta",
"IN:te",
"IT:it",
"JP:ja",
"KE:en",
"KR:ko",
"LB:ar",
"LT:lt",
"LV:en",
"LV:lv",
"MA:fr",
"MY:en",
"MY:ms",
"NA:en",
"NG:en",
"NL:nl",
"NO:no",
"NZ:en",
"PH:en",
"PK:en",
"PL:pl",
"RO:ro",
"RS:sr",
"RU:ru",
"SA:ar",
"SE:sv",
"SG:en",
"SI:sl",
"SK:sk",
"SN:fr",
"TH:th",
"TR:tr",
"TZ:en",
"UA:ru",
"UA:uk",
"UG:en",
"US:en",
"VN:vi",
"ZA:en",
"ZW:en",
]
"""List of region/language combinations supported by Google News. Values of the
``ceid`` argument of the Google News REST API."""
_skip_values = [
"ET:en", # english (ethiopia)
"ID:en", # english (indonesia)
"LV:en", # english (latvia)
]
_ceid_locale_map = {"NO:no": "nb-NO"}
def fetch_traits(engine_traits: EngineTraits):
_fetch_traits(engine_traits, add_domains=False)
engine_traits.custom["ceid"] = {}
for ceid in ceid_list:
if ceid in _skip_values:
continue
region, lang = ceid.split(":")
x = lang.split("-")
if len(x) > 1:
if x[1] not in ["Hant", "Hans"]:
lang = x[0]
sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
try:
locale = babel.Locale.parse(sxng_locale, sep="-")
except babel.UnknownLocaleError:
print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
continue
engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid

1
searx/engines/google_play.py
View File

@@ -20,7 +20,6 @@ about = {
}
play_categ = None # apps|movies
enable_http3 = True
base_url = 'https://play.google.com'
search_url = base_url + "/store/search?{query}&c={play_categ}"

9
searx/engines/google_scholar.py
View File

@@ -27,7 +27,7 @@ import typing as t
from urllib.parse import urlencode
from datetime import datetime
from lxml import html
from curl_cffi.requests.exceptions import TooManyRedirects
import httpx
from searx.utils import (
eval_xpath,
@@ -63,7 +63,6 @@ about = {
# engine dependent config
categories = ["science", "scientific publications"]
paging = True
enable_http3 = True
max_page = 50
"""`Google max 50 pages`_
@@ -78,6 +77,8 @@ def request(query: str, params: "OnlineParams") -> None:
"""Google-Scholar search request"""
google_info = get_google_info(params, traits)
# subdomain is: scholar.google.xy
google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
args = {
"q": query,
@@ -88,7 +89,7 @@ def request(query: str, params: "OnlineParams") -> None:
}
args.update(time_range_args(params))
params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
@@ -103,7 +104,7 @@ def response(resp: "SXNG_Response") -> EngineResults: # pylint: disable=too-man
raise SearxEngineAccessDeniedException(
message="google_scholar: unusual traffic detected",
)
raise TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
raise httpx.TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
res = EngineResults()
dom = html.fromstring(resp.text)

204
searx/engines/google_videos.py
View File

@@ -1,85 +1,185 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Videos: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Videos engine.
import typing as t
.. admonition:: Content-Security-Policy (CSP)
This engine needs to allow images from the `data URLs`_ (prefixed with the
``data:`` scheme)::
Header set Content-Security-Policy "img-src 'self' data: ;"
.. _data URLs:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
"""
import re
from urllib.parse import urlencode, urlparse, parse_qs, unquote
from lxml import html
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
parse_duration_string,
)
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
time_range_dict,
filter_mapping,
suggestion_xpath,
detect_google_sorry,
ui_async,
)
from searx.utils import get_embeded_stream_url
# about
about = {
"website": "https://www.google.com",
"wikidata_id": "Q219885",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://www.google.com',
"wikidata_id": 'Q219885',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'HTML',
}
# engine dependent config
categories = ["videos", "web"]
categories = ['videos', 'web']
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
language_support = True
time_range_support = True
safesearch = True
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "vid"},
eng_traits=traits,
use_locales=False,
# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
def parse_data_images(text: str):
data_image_map = {}
for img_id, data_image in RE_DATA_IMAGE.findall(text):
end_pos = data_image.rfind("=")
if end_pos > 0:
data_image = data_image[: end_pos + 1]
data_image_map[img_id] = data_image
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def request(query, params):
"""Google-Video search request"""
google_info = get_google_info(params, traits)
start = (params['pageno'] - 1) * 10
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ "?"
+ urlencode(
{
'q': query,
'tbm': "vid",
'start': start,
**google_info['params'],
'asearch': 'arc',
'async': ui_async(start),
}
)
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if 'safesearch' in params:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
return params
for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
data_image_map = parse_data_images(resp.text)
# convert the text to dom
dom = html.fromstring(resp.text)
result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
# parse results
for result in result_divs:
title = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
allow_none=True,
)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
if not title or not raw_url:
continue
url = unwrap_google_url(raw_url)
thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
if "/default.jpg" in thumbnail:
thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
length = None
for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
length = parse_duration_string(extract_text(span) or "")
if length:
break
results.add(
results.types.MainResult(
url=url,
title=title,
thumbnail=thumbnail,
length=length,
template="videos.html",
)
url = eval_xpath_getindex(
result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
)
if url and url.startswith('/url?q='):
url = unquote(url[7:].split('&sa=U')[0])
content = extract_text(
eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
)
pub_info = extract_text(
eval_xpath_getindex(
result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
),
allow_none=True,
)
# Broader XPath to find any <img> element
thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
duration = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
)
video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
# Fallback for video_id from URL if not found via XPath
if not video_id and url and 'youtube.com' in url:
parsed_url = urlparse(url)
video_id = parse_qs(parsed_url.query).get('v', [None])[0]
# Handle thumbnail
if thumbnail and thumbnail.startswith('data:image'):
img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
if img_id and img_id in data_image_map:
thumbnail = data_image_map[img_id]
else:
thumbnail = None
if not thumbnail and video_id:
thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
# Handle video embed URL
embed_url = None
if video_id:
embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
elif url:
embed_url = get_embeded_stream_url(url)
# Only append results with valid title and url
if title and url:
results.append(
{
'url': url,
'title': title,
'content': content or '',
'author': pub_info,
'thumbnail': thumbnail,
'length': duration,
'iframe_src': embed_url,
'template': 'videos.html',
}
)
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.append({'suggestion': extract_text(suggestion)})
return results

127
searx/engines/heexy.py Normal file
View File

@@ -0,0 +1,127 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Heexy_ is a minimalist search engine that focuses on privacy.
Although it also supports news and videos, these are not implemented here
because they usually return no result to very few irrelevant ones.
It seems to use Bing internally, as the image thumbnails are loaded from Bing.
.. _Heexy: https://docs.heexy.org/introduction
"""
from urllib.parse import urlencode
import typing as t
from searx.enginelib import EngineCache
from searx.network import get
from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
from searx.result_types import EngineResults
from searx.utils import gen_useragent
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
about = {
"website": "https://heexy.org",
"wikidata_id": None,
"official_api_documentation": None,
"use_official_api": False,
"require_api_key": False,
"results": "JSON",
}
paging = True
safesearch = True
categories = ["general"]
heexy_categ = "web"
"""Category to search in. Can be either "web" or "image"."""
base_url = "https://heexy.org"
api_url = "https://seapi.heexy.org"
safe_search_map = {0: "off", 1: "on", 2: "on"}
CACHE: EngineCache
"""Cache for storing the ``X-Data-Cacheft`` token (acts like an API key)."""
def setup(engine_settings: dict[str, t.Any]) -> bool:
global CACHE # pylint: disable=global-statement
if heexy_categ not in ("web", "image"):
raise ValueError("invalid search category: %s" % heexy_categ)
CACHE = EngineCache(engine_settings["name"])
return True
def _get_api_token(query: str) -> str:
"""The API token is independent of the search query. We just need any query
to obtain it initially, and don't hardcode it here to decrease chances of
getting blocked. The token must be passed as ``X-Data-Cacheft`` header."""
cached_token: str = CACHE.get("token")
if cached_token:
return cached_token
resp = get(
f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent(), "Accept-Language": "en-US,en:q=0.9"}
)
if not resp.ok:
raise SearxEngineAPIException("failed to obtain request token: invalid response code")
token = resp.cookies["cacheft"]
if not token:
raise SearxEngineAPIException("failed to obtain request token: no token found")
CACHE.set("token", token, expire=3 * 60)
return token
def request(query: str, params: "OnlineParams") -> None:
args = {
"q": query,
"page": params["pageno"],
"safe": safe_search_map[params["safesearch"]],
}
if params["searxng_locale"] != "all":
args["lang"] = params["searxng_locale"].split("-")[0]
params["url"] = f"{api_url}/search/{heexy_categ}?{urlencode(args)}"
params["headers"]["Origin"] = api_url
params["cookies"]["cacheft"] = _get_api_token(query)
def response(resp: "SXNG_Response"):
res = EngineResults()
json_resp = resp.json()
if not json_resp["success"]:
raise SearxEngineAccessDeniedException()
result: dict[str, str]
for result in json_resp["results"]:
if heexy_categ == "web":
res.add(
res.types.MainResult(
url=result["url"],
title=result["title"],
content=result["description"],
)
)
elif heexy_categ == "image":
res.add(
res.types.Image(
title=result["description"],
url=result["url"],
thumbnail_src=result["image"],
img_src=result["rawImage"],
)
)
return res

3
searx/engines/huggingface.py
View File

@@ -33,7 +33,6 @@ Implementations
"""
import typing as t
from urllib.parse import urlencode
from datetime import datetime
@@ -63,7 +62,7 @@ huggingface_endpoint = 'models'
"""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if huggingface_endpoint not in ('datasets', 'models', 'spaces'):
raise SearxEngineAPIException(f"Unsupported Hugging Face endpoint: {huggingface_endpoint}")

3
searx/engines/invidious.py
View File

@@ -8,7 +8,6 @@ No public instance offer a public API now
"""
import typing as t
import time
import random
from urllib.parse import quote_plus, urlparse
@@ -35,7 +34,7 @@ time_range_support = True
base_url: list[str] | str = []
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not base_url:
raise ValueError("missing invidious base_url")

2
searx/engines/kagi.py
View File

@@ -87,7 +87,7 @@ api_key = ""
"""Kagi API key. Required for using this engine."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not api_key:
raise ValueError("api_key is required for using kagi")

8
searx/engines/luxxle.py
View File

@@ -37,17 +37,21 @@ about = {
}
categories = []
safesearch = True
safeseach = True
base_url = "https://luxxle.com"
luxxle_categ = "search"
"""Supported categories: "search", "news", "images", "videos"."""
# otherwise all requests get blocked (http2-fingerprinted probably)
enable_http2 = False
safe_search_map = {0: "Off", 1: "Moderate", 2: "Strict"}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if luxxle_categ not in ("search", "images", "videos", "news"):
raise ValueError("invalid luxxle category: %s" % luxxle_categ)

6
searx/engines/marginalia.py
View File

@@ -10,7 +10,7 @@ Lofgren .
.. _marginalia filters:
Marginalia Filters
==================
=================
Custom filters enable server-side customization of Marginalia search results.
Filter definitions are written in XML and scoped to an API key. Filters can
@@ -82,7 +82,7 @@ api_key = None
https://about.marginalia-search.com/article/api/
"""
filter_name: str = ""
filter_name: str | None = None
"""The name of the custom filter to apply to each search."""
@@ -164,7 +164,7 @@ def response(resp: SXNG_Response):
return res
def setup(_: dict[str, t.Any]) -> bool:
def init(_: dict[str, t.Any]):
if not api_key:
logger.error("missing api_key: see https://about.marginalia-search.com/article/api")

10
searx/engines/mariadb_server.py
View File

@@ -26,8 +26,6 @@ Implementations
"""
import typing as t
try:
import mariadb # pyright: ignore [reportMissingImports]
except ImportError:
@@ -62,17 +60,15 @@ paging = True
_connection = None
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
global _connection # pylint: disable=global-statement
if 'query_str' not in engine_settings:
raise ValueError('query_str cannot be empty')
if not engine_settings['query_str'].lower().startswith('select '):
raise ValueError('only SELECT query is supported')
def init(_):
global _connection # pylint: disable=global-statement
_connection = mariadb.connect(database=database, user=username, password=password, host=host, port=port)

4
searx/engines/meilisearch.py
View File

@@ -35,7 +35,7 @@ Here is a simple example to query a Meilisearch instance:
"""
# pylint: disable=global-statement
import typing as t
from json import dumps
from searx.result_types import EngineResults
from searx.extended_types import SXNG_Response
@@ -49,7 +49,7 @@ categories = ['general']
paging = True
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if index == '':
raise ValueError('index cannot be empty')

3
searx/engines/mojeek.py
View File

@@ -1,7 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Mojeek (general, images, news)"""
import typing as t
from datetime import datetime
from urllib.parse import urlencode
@@ -52,7 +51,7 @@ region_param = "arc"
_delta_kwargs = {"day": "days", "week": "weeks", "month": "months", "year": "years"}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if search_type not in ("", "images", "news"):
raise ValueError(f"Invalid search type {search_type}")

3
searx/engines/mrs.py
View File

@@ -19,7 +19,6 @@ Implementation
==============
"""
import typing as t
from urllib.parse import quote_plus
about = {
@@ -38,7 +37,7 @@ matrix_url = "https://matrix.to"
page_size = 20
def setup(_: dict[str, t.Any]) -> bool | None:
def init(engine_settings): # pylint: disable=unused-argument
"""The ``base_url`` must be set in the configuration, if ``base_url`` is not
set, a :py:obj:`ValueError` is raised during initialization.

10
searx/engines/mysql_server.py
View File

@@ -25,8 +25,6 @@ Implementations
"""
import typing as t
from searx.result_types import EngineResults
try:
@@ -62,17 +60,15 @@ paging = True
_connection = None
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
global _connection # pylint: disable=global-statement
if 'query_str' not in engine_settings:
raise ValueError('query_str cannot be empty')
if not engine_settings['query_str'].lower().startswith('select '):
raise ValueError('only SELECT query is supported')
def init(_):
global _connection # pylint: disable=global-statement
_connection = mysql.connector.connect(
database=database,
user=username,

26
searx/engines/naver.py
View File

@@ -2,8 +2,6 @@
# pylint: disable=line-too-long
"""Naver for SearXNG"""
import typing as t
from urllib.parse import urlencode
from lxml import html
@@ -18,6 +16,7 @@ from searx.utils import (
html_to_text,
parse_duration_string,
js_obj_str_to_python,
get_embeded_stream_url,
)
# engine metadata
@@ -68,7 +67,7 @@ naver_category_dict = {
}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if naver_category not in ('general', 'images', 'news', 'videos'):
raise SearxEngineAPIException(f"Unsupported category: {naver_category}")
@@ -194,7 +193,7 @@ def parse_news(data):
def parse_videos(data):
res = EngineResults()
results = []
dom = html.fromstring(data)
@@ -213,14 +212,15 @@ def parse_videos(data):
except (ValueError, TypeError):
pass
res.add(
res.types.LegacyResult(
template="videos.html",
title=extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
url=url,
thumbnail=thumbnail,
length=length,
)
results.append(
{
"template": "videos.html",
"title": extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
"url": url,
"thumbnail": thumbnail,
"length": length,
"iframe_src": get_embeded_stream_url(url),
}
)
return res
return results

26
searx/engines/neosearch.py
View File

@@ -6,14 +6,10 @@
from json import loads
import typing as t
from urllib.parse import urlencode
from lxml import html
from searx.exceptions import SearxEngineAPIException
from searx.extended_types import SXNG_Response
from searx.network import get
from searx.result_types import EngineResults
from searx.utils import eval_xpath, extract_text
if t.TYPE_CHECKING:
from searx.enginelib.traits import EngineTraits
@@ -29,33 +25,18 @@ about = {
"results": "JSON",
}
paging = False
enable_http3 = True
base_url = "https://neosearch.org"
categories = ["general"]
def _obtain_xsrf_token() -> str:
resp = get(base_url)
doc = html.fromstring(resp.text)
xsrf_token = extract_text(eval_xpath(doc, "//meta[@name='xsrf-token']/@content"))
if not xsrf_token:
raise SearxEngineAPIException("failed to obtain xsrf token")
return xsrf_token
paging = False
def request(query: str, params: "OnlineParams"):
params["url"] = f"{base_url}/search"
params["headers"]["X-XSRF-TOKEN"] = _obtain_xsrf_token()
params["method"] = "POST"
args = {"q": query, "generate": "auto"}
countrycode = params["searxng_locale"].split("-")[-1].upper()
if countrycode in traits.custom["countrycodes"]:
args["loc"] = countrycode
params["json"] = args
params["url"] = f"{base_url}/search?{urlencode(args)}"
def response(resp: "SXNG_Response") -> EngineResults:
@@ -86,6 +67,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
def fetch_traits(engine_traits: "EngineTraits") -> None:
# pylint: disable=import-outside-toplevel
from searx.network import get
from searx.utils import extr, js_obj_str_to_python
from babel.core import get_global

6
searx/engines/openstreetmap.py
View File

@@ -10,8 +10,7 @@ from flask_babel import gettext
from searx.data import OSM_KEYS_TAGS, CURRENCIES
from searx.external_urls import get_external_url
from searx.wikidata import send_wikidata_query
from searx.engines.wikidata import sparql_string_escape, get_thumbnail
from searx.engines.wikidata import send_wikidata_query, sparql_string_escape, get_thumbnail
from searx.result_types import EngineResults
# about
@@ -291,8 +290,7 @@ def get_title_address(result):
'house_number': address_raw.get('house_number'),
'road': address_raw.get('road'),
'locality': address_raw.get(
'city',
address_raw.get('town', address_raw.get('village')), # noqa
'city', address_raw.get('town', address_raw.get('village')) # noqa
), # noqa
'postcode': address_raw.get('postcode'),
'country': address_raw.get('country'),

18
searx/engines/pexels.py
View File

@@ -2,13 +2,12 @@
"""Pexels (images)"""
import re
import typing as t
from urllib.parse import urlencode
from lxml import html
from searx.result_types import EngineResults
from searx.utils import eval_xpath_list
from searx.utils import eval_xpath_list, gen_useragent
from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
from searx.network import get
@@ -44,17 +43,24 @@ SECRET_KEY_DB_KEY = "secret-key"
CACHE: EngineCache
"""Cache to store the secret API key for the engine."""
enable_http2 = False
def setup(engine_settings: dict[str, t.Any]) -> bool:
def init(engine_settings):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
return True
def _get_secret_key():
resp = get(
base_url,
headers={"Referer": base_url},
headers={
# circumvents Cloudflare bot protections
"User-Agent": gen_useragent(),
"Referer": base_url,
"Sec-GPC": "1",
"Connection": "keep-alive",
},
)
if resp.status_code != 200:
@@ -97,6 +103,8 @@ def request(query, params):
params["headers"]["secret-key"] = secret_key
return params
def response(resp):
res = EngineResults()

5
searx/engines/pixabay.py
View File

@@ -27,6 +27,9 @@ time_range_support = True
safesearch_map = {0: 'off', 1: '1', 2: '1'}
time_range_map = {'day': '1d', 'week': '1w', 'month': '1m', 'year': '1y'}
# using http2 returns forbidden errors
enable_http2 = False
def request(query, params):
args = {
@@ -47,6 +50,8 @@ def request(query, params):
# prevent automatic redirects to first page on pagination
params['allow_redirects'] = False
return params
def _image_result(result):
return {

10
searx/engines/postgresql.py
View File

@@ -21,8 +21,6 @@ Implementations
"""
import typing as t
try:
import psycopg2 # type: ignore
except ImportError:
@@ -57,17 +55,15 @@ paging = True
_connection = None
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
global _connection # pylint: disable=global-statement
if 'query_str' not in engine_settings:
raise ValueError('query_str cannot be empty')
if not engine_settings['query_str'].lower().startswith('select '):
raise ValueError('only SELECT query is supported')
def init(_):
global _connection # pylint: disable=global-statement
_connection = psycopg2.connect(
database=database,
user=username,

5
searx/engines/privacywall.py
View File

@@ -14,7 +14,7 @@ from lxml import html
import babel
from searx.enginelib.traits import EngineTraits
from searx.utils import eval_xpath_list, eval_xpath, extract_text, extr
from searx.utils import eval_xpath_list, eval_xpath, extract_text, get_embeded_stream_url, extr
from searx.locales import region_tag
from searx.result_types import EngineResults
@@ -61,7 +61,7 @@ video_page_map = {
}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if privacywall_category not in ("general", "images", "videos"):
raise ValueError("invalid category: %s" % privacywall_category)
@@ -154,6 +154,7 @@ def _video_results(doc: "ElementBase") -> EngineResults:
title=extract_text(eval_xpath(result, ".//h2[contains(@class, 'video-card-title')]")) or "",
content=extract_text(eval_xpath(result, ".//p")) or "",
thumbnail=thumbnail or "",
iframe_src=get_embeded_stream_url(url) or "",
)
)

3
searx/engines/quark.py
View File

@@ -1,7 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Quark (Shenma) search engine for searxng"""
import typing as t
from urllib.parse import urlencode
from datetime import datetime
import re
@@ -44,7 +43,7 @@ def is_alibaba_captcha(html):
return bool(re.search(CAPTCHA_PATTERN, html))
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if quark_category not in ('general', 'images'):
raise SearxEngineAPIException(f"Unsupported category: {quark_category}")

4
searx/engines/qwant.py
View File

@@ -60,6 +60,9 @@ from searx.exceptions import (
SearxEngineTooManyRequestsException,
)
from searx.network import raise_for_httperror
from searx.utils import (
get_embeded_stream_url,
)
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
@@ -296,6 +299,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
title=title,
url=res_url,
content=content,
iframe_src=get_embeded_stream_url(res_url),
publishedDate=pub_date,
thumbnail=thumbnail,
template="videos.html",

9
searx/engines/resulthunter.py
View File

@@ -10,7 +10,6 @@ from urllib.parse import urlencode
from lxml import html
from searx import locales
from searx.exceptions import SearxEngineResponseException
from searx.result_types import EngineResults
from searx.utils import eval_xpath_list, eval_xpath, extract_text
@@ -43,7 +42,7 @@ resulthunter_categ = "web"
"""Supported categories are ``web`` and ``images``."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if resulthunter_categ not in ("web", "images"):
raise ValueError("invalid category: %s" % resulthunter_categ)
@@ -53,7 +52,6 @@ def request(query: str, params: "OnlineParams") -> None:
"q": query,
"search_type": resulthunter_categ,
"offset": params["pageno"] - 1,
"search_source": "other",
}
# uses Brave's engine traits
@@ -113,11 +111,6 @@ def _image_results(doc: "ElementBase") -> EngineResults:
def response(resp: "SXNG_Response") -> EngineResults:
doc = html.fromstring(resp.text)
# if the request was wrong (e.g. missing params), the site doesn't contain a result container
# and instead shows an "Installation required" page to download the resulthunter browser extension
if not eval_xpath(doc, "//div[contains(@class, 'organic-results-container')]"):
raise SearxEngineResponseException()
match resulthunter_categ:
case "web":
return _general_results(doc)

6
searx/engines/s1search.py
View File

@@ -38,10 +38,12 @@ CACHE: EngineCache
"""Cache to store verification tokens for pagination."""
def setup(engine_settings: dict[str, t.Any]) -> bool:
def init(_):
if not base_url:
raise ValueError("base_url must be set")
def setup(engine_settings: dict[str, t.Any]) -> bool:
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
return True
@@ -80,7 +82,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
res.types.MainResult(
url=extract_text(eval_xpath(result, ".//a[contains(@class, 'title')]/@href")),
title=extract_text(eval_xpath(result, ".//a[contains(@class, 'title')]")),
content=extract_text(eval_xpath(result, ".//span[contains(@class, 'description') or not(@class)]")),
content=extract_text(eval_xpath(result, ".//span[contains(@class, 'description') or @class='']")),
)
)

63
searx/engines/s1search_rampjs.py
View File

@@ -1,63 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""JavaScript-based s1search implementation. See :ref:`s1search engine`.
Works for all s1search sites that contain the ``__RAMPJS__`` JavaScript variable.
"""
import json
import typing as t
from urllib.parse import urlencode
from searx.result_types import EngineResults
from searx.utils import extr, html_to_text
if t.TYPE_CHECKING:
from searx.search.processors import OnlineParams
from searx.extended_types import SXNG_Response
about = {
"website": "https://s1search.co",
"official_api_documentation": None,
"use_official_api": False,
"require_api_key": False,
"results": "JSON",
}
categories = ["general"]
paging = True
base_url = "https://search.answers.com"
# other working base URLs:
# - https://search.nation.online
# - https://search.activebeat.com
# - https://search.legalboulevard.com
# - https://search.walletgenius.com
# - https://search.legalboulevard.com
def request(query: str, params: "OnlineParams"):
args = {"q": query, "page": params["pageno"]}
params["url"] = f"{base_url}/?{urlencode(args)}"
def response(resp: "SXNG_Response") -> EngineResults:
res = EngineResults()
data_raw = extr(resp.text, "response: ", " };")
data = json.loads(data_raw)
mainline = [s for s in data["search"]["regions"] if s["name"] == "mainline"][0]
for group in mainline["groups"]:
for result in group["results"]:
if not ("url" in result or "clickUrl" in result):
continue
res.add(
res.types.MainResult(
url=result.get("url") or result.get("clickUrl"),
title=html_to_text(result["title"]),
content=html_to_text(result["description"]),
)
)
return res

88
searx/engines/searchrockit.py
View File

@@ -1,88 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""SearchRockit is an American search engine. It allegedly has its own index,
but the results seem to come from Google."""
import typing as t
from urllib.parse import urlencode
from lxml import html
from dateutil import parser
from searx.result_types import EngineResults
from searx.utils import (
eval_xpath_list,
extract_text,
eval_xpath,
)
if t.TYPE_CHECKING:
from searx.search.processors import OnlineParams
from searx.extended_types import SXNG_Response
about = {
"website": "https://searchrockit.com",
"official_api_documentation": None,
"use_official_api": False,
"require_api_key": False,
"results": "HTML",
}
categories = ["general"]
paging = True
SearchrockitCateg = t.Literal["web", "images", "news"]
searchrockit_categ: SearchrockitCateg = "web"
base_url = "https://searchrockit.com"
def setup(_):
if searchrockit_categ not in t.get_args(SearchrockitCateg):
raise ValueError("invalid search category: %s" % searchrockit_categ)
def request(query: str, params: "OnlineParams") -> None:
args = {"q": query, "p": params["pageno"]}
params["url"] = f"{base_url}/results/{searchrockit_categ}?{urlencode(args)}"
def response(resp: "SXNG_Response") -> EngineResults:
doc = html.fromstring(resp.text)
res = EngineResults()
match searchrockit_categ:
case "web" | "news":
for result in eval_xpath_list(
doc, "//div[contains(@class, 'results-list')]/div[contains(@class, 'result-item')]"
):
publishedDate = None
try:
d = extract_text(eval_xpath(result, ".//span[contains(@class, 'result-item--publishedAt')]")) or ""
publishedDate = parser.parse(d)
except parser.ParserError:
pass
res.add(
res.types.MainResult(
url=extract_text(eval_xpath(result, ".//a[contains(@class, 'result-item--title')]/@href")),
title=extract_text(eval_xpath(result, ".//a[contains(@class, 'result-item--title')]")) or "",
content=extract_text(eval_xpath(result, ".//a[contains(@class, 'result-item--desc')]")) or "",
thumbnail=extract_text(
eval_xpath(result, ".//a[contains(@class, 'result-item--thumb')]/img/@src")
)
or "",
publishedDate=publishedDate,
)
)
case "images":
for result in eval_xpath_list(
doc, "//div[contains(@class, 'image-grid')]/a[contains(@class, 'image-card')]"
):
res.add(
res.types.Image(
url=extract_text(eval_xpath(result, "./@href")),
title=extract_text(eval_xpath(result, "./div[contains(@class, 'image-title')]")) or "",
thumbnail_src=extract_text(eval_xpath(result, "./img/@src")) or "",
img_src=extract_text(eval_xpath(result, "./@data-full-url")) or "",
)
)
return res

34
searx/engines/searchzee.py
View File

@@ -4,11 +4,13 @@ independent search infrastructure."""
import typing as t
from urllib.parse import urlencode
import uuid
from searx.exceptions import SearxEngineAPIException
from searx.extended_types import SXNG_Response
from searx.network import get
from searx.result_types import EngineResults
from searx.utils import html_to_text
from searx.utils import extr, html_to_text
from searx.enginelib import EngineCache
if t.TYPE_CHECKING:
from searx.search.processors import OnlineParams
@@ -32,19 +34,43 @@ SearchzeeCategType = t.Literal["web", "news"]
searchzee_categ: SearchzeeCategType = None # type: ignore[reportAssignmentType]
CACHE: EngineCache
"""Cache for storing the scraped API Token."""
base_url = "https://searchzee.com"
# only supports for news
time_range_map = {"day": "pd", "week": "pw", "month": "pm", "year": "py"}
def setup(_: dict[str, t.Any]):
def setup(engine_settings: dict[str, t.Any]) -> bool:
if searchzee_categ not in t.get_args(SearchzeeCategType):
raise ValueError("invalid category: %s" % searchzee_categ)
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"]) # type: ignore[reportAny]
return True
def _obtain_api_token() -> str:
token: str | None = CACHE.get("token") # type: ignore[reportAny]
if token:
return token
token_resp = get(
f"{base_url}/app.js",
)
if not token_resp.ok:
raise SearxEngineAPIException("failed to obtain api key")
token = extr(token_resp.text, "const SEARCHZEE_API_TOKEN = \"", "\";")
CACHE.set("token", token, expire=3600)
return token
def request(query: str, params: "OnlineParams"):
params["cookies"]["szs"] = str(uuid.uuid4())
params["headers"]["X-SearchZee-Token"] = _obtain_api_token()
args = {"q": query, "type": searchzee_categ, "offset": params["pageno"] - 1}
if params["time_range"]:

57
searx/engines/searx_engine.py Normal file
View File

@@ -0,0 +1,57 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""
Searx (all)
"""
from json import loads
from searx.engines import categories as searx_categories
# about
about = {
"website": 'https://github.com/searxng/searxng',
"wikidata_id": 'Q17639196',
"official_api_documentation": 'https://docs.searxng.org/dev/search_api.html',
"use_official_api": True,
"require_api_key": False,
"results": 'JSON',
}
categories = searx_categories.keys()
# search-url
instance_urls = []
instance_index = 0
# do search-request
def request(query, params):
global instance_index # pylint: disable=global-statement
params['url'] = instance_urls[instance_index % len(instance_urls)]
params['method'] = 'POST'
instance_index += 1
params['data'] = {
'q': query,
'pageno': params['pageno'],
'language': params['language'],
'time_range': params['time_range'],
'category': params['category'],
'format': 'json',
}
return params
# get response from search-request
def response(resp):
response_json = loads(resp.text)
results = response_json['results']
for i in ('answers', 'infoboxes'):
results.extend(response_json[i])
results.extend({'suggestion': s} for s in response_json['suggestions'])
return results

4
searx/engines/solr.py
View File

@@ -28,7 +28,7 @@ This is an example configuration for searching in the collection
"""
# pylint: disable=global-statement
import typing as t
from urllib.parse import urlencode
from searx.exceptions import SearxEngineAPIException
from searx.result_types import EngineResults
@@ -46,7 +46,7 @@ _search_url = ''
paging = True
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if collection == '':
raise ValueError('collection cannot be empty')

4
searx/engines/soundcloud.py
View File

@@ -1,7 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""SoundCloud is a German audio streaming service."""
import typing as t
import re
import datetime
@@ -116,10 +115,9 @@ def response(resp):
return results
def setup(engine_settings: dict[str, t.Any]) -> bool:
def init(engine_settings): # pylint: disable=unused-argument
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"]) # type:ignore
return True
def get_client_id() -> str | None:

2
searx/engines/sqlite.py
View File

@@ -66,7 +66,7 @@ limit = 10
paging = True
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
def init(engine_settings):
if 'query_str' not in engine_settings:
raise ValueError('query_str cannot be empty')

72
searx/engines/startpage.py
View File

@@ -1,8 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Startpage requires solving an Anubis POW captcha (difficulty 4).
Solving it requires a lot of CPU, so the engine is set inactive by default.
Startpage's language & region selectors are a mess ..
"""Startpage's language & region selectors are a mess ..
.. _startpage regions:
@@ -87,7 +84,6 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
"""
# pylint: disable=too-many-statements
import hashlib
import re
import typing as t
from collections import OrderedDict
@@ -103,7 +99,7 @@ from searx.enginelib import EngineCache
from searx.enginelib.traits import EngineTraits
from searx.exceptions import SearxEngineCaptchaException
from searx.locales import region_tag
from searx.network import get, post # see https://github.com/searxng/searxng/issues/762
from searx.network import get # see https://github.com/searxng/searxng/issues/762
from searx.utils import (
eval_xpath,
extr,
@@ -169,7 +165,7 @@ CACHE: EngineCache
seconds."""
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
global CACHE # pylint: disable=global-statement
# hint: all three startpage engines (WEB, Images & News) can/should use the
@@ -180,45 +176,6 @@ def setup(_: dict[str, t.Any]) -> bool | None:
sc_code_cache_sec = 3600
"""Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
# startpage's anubis difficulty is set to 4
max_difficulty = 4
def _solve_anubis(resp) -> str:
"""Anubis POW solver"""
payload = loads(extr(resp.text, '<script id="anubis_challenge" type="application/json">', "</script>"))
challenge = payload["challenge"]
difficulty = int(payload["rules"]["difficulty"])
if difficulty > max_difficulty:
raise SearxEngineCaptchaException(message="startpage: Anubis difficulty too high")
prefix = "0" * difficulty
blob = challenge["randomData"].encode()
for nonce in range(16**difficulty * 8): # max search is 8x average search, e^-8 = 0.034% will fail
digest = hashlib.sha256(blob + str(nonce).encode()).hexdigest()
if digest.startswith(prefix):
break
else:
raise SearxEngineCaptchaException(message="startpage: Anubis failed")
pass_resp = get(
f"{base_url}/.within.website/x/cmd/anubis/api/pass-challenge",
params={
"id": challenge["id"],
"response": digest,
"nonce": nonce,
"redir": str(resp.url),
"elapsedTime": "1",
},
cookies=resp.cookies,
allow_redirects=False,
)
auth = pass_resp.cookies.get("spchal-auth")
if not auth:
raise SearxEngineCaptchaException(message="startpage: Anubis pass-challenge failed")
auth = str(auth)
CACHE.set("SPCHAL_AUTH", auth, expire=240)
return auth
def get_sc_code(params):
"""Get an actual ``sc`` argument from Startpage's search form (HTML page).
@@ -244,9 +201,6 @@ def get_sc_code(params):
logger.debug("get_sc_code: request headers: %s", headers)
resp = get(get_sc_url, headers=headers)
if 'id="anubis_challenge"' in resp.text:
resp = get(get_sc_url, headers=headers, cookies={"spchal-auth": _solve_anubis(resp)})
# ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)
# ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg
# ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21
@@ -285,8 +239,8 @@ def request(query, params):
Additionally the arguments form Startpage's search form needs to be set in
HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.
"""
engine_region = traits.get_region(params["searxng_locale"], "en_US")
engine_language = traits.get_language(params["searxng_locale"], "english")
engine_region = traits.get_region(params["searxng_locale"], "en-US")
engine_language = traits.get_language(params["searxng_locale"], "en")
params["headers"]["Origin"] = base_url
params["headers"]["Referer"] = base_url + "/"
@@ -308,9 +262,9 @@ def request(query, params):
args["language"] = engine_language
args["lui"] = engine_language
args["segment"] = "startpage.udog"
if params["pageno"] > 1:
args["page"] = params["pageno"]
args["segment"] = "startpage.udog"
# Build cookie
lang_homepage = "en"
@@ -335,8 +289,6 @@ def request(query, params):
cookie["search_results_region"] = engine_region
params["cookies"]["preferences"] = "N1N".join(["%sEEE%s" % x for x in cookie.items()])
if auth := CACHE.get("SPCHAL_AUTH"):
params["cookies"]["spchal-auth"] = auth
logger.debug("cookie preferences: %s", params["cookies"]["preferences"])
logger.debug("data: %s", args)
@@ -448,18 +400,6 @@ def _get_image_result(result) -> dict[str, t.Any] | None:
def response(resp):
if 'id="anubis_challenge"' in resp.text:
params = resp.search_params
params["cookies"]["spchal-auth"] = _solve_anubis(resp)
resp = post(
params["url"] or search_url,
data=params["data"],
headers=params["headers"],
cookies=params["cookies"],
)
if 'id="anubis_challenge"' in resp.text:
raise SearxEngineCaptchaException()
categ = startpage_categ.capitalize()
results_raw = "{" + extr(resp.text, f"React.createElement(UIStartpage.AppSerp{categ}, {{", "}})") + "}}"

2
searx/engines/startpagina.py
View File

@@ -39,7 +39,7 @@ page_size = 10
api_url = "https://search.kompas.services"
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if startpagina_categ not in ("web", "images", "videos", "news"):
raise ValueError("invalid search type: %s" % startpagina_categ)

2
searx/engines/stocksnap.py
View File

@@ -21,6 +21,8 @@ about = {
"require_api_key": False,
"results": "JSON",
}
# otherwise all requests get blocked, probably HTTP2 fingerprinting
enable_http2 = False
base_url = "https://stocksnap.io"
cdn_url = "https://cdn.stocksnap.io"

2
searx/engines/swisscows.py
View File

@@ -152,7 +152,7 @@ def generate_nonce_and_signature(base_path: str, args: dict[str, t.Any]) -> tupl
maximum_page_size = {"web": 20, "images": 50, "videos": 10}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if swisscows_category not in ("web", "images", "videos"):
raise ValueError("illegal swisscows category: %s" % swisscows_category)

4
searx/engines/tiger.py
View File

@@ -45,10 +45,12 @@ CACHE: EngineCache
"""Cache to store session codes (result of solved CAPTCHA)."""
def setup(engine_settings: dict[str, t.Any]) -> bool:
def init(_):
if tiger_category not in ("Websuche", "News"):
raise ValueError("invalid search category: %s" % tiger_category)
def setup(engine_settings: dict[str, t.Any]) -> bool:
global CACHE # pylint: disable=global-statement
CACHE = EngineCache(engine_settings["name"])
return True

5
searx/engines/tonline.py
View File

@@ -14,7 +14,7 @@ from urllib.parse import urlencode
from lxml import html
from searx.utils import eval_xpath_list, eval_xpath, extract_text, ElementType
from searx.utils import eval_xpath_list, eval_xpath, extract_text, get_embeded_stream_url, ElementType
from searx.result_types import EngineResults
from searx.enginelib import EngineAbout
@@ -44,7 +44,7 @@ tonline_channel_map = {"images": "flickr", "videos": "yt"}
language = "de"
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if tonline_categ not in ("web", "images", "videos", "news"):
raise ValueError("invalid category: %s" % tonline_categ)
@@ -126,6 +126,7 @@ def _video_results(doc: ElementType, res: EngineResults):
url=url,
title=" - ".join(extract_text(part) or "" for part in title_parts),
thumbnail=extract_text(eval_xpath(result, ".//img/@src") or "") or "",
iframe_src=get_embeded_stream_url(url) or "",
)
)

2
searx/engines/torznab.py
View File

@@ -83,7 +83,7 @@ show_torrent_files: bool = False
show_magnet_links: bool = True
def setup(_: dict[str, t.Any]) -> bool | None:
def init(engine_settings=None): # pylint: disable=unused-argument
"""Initialize the engine."""
if len(base_url) < 1:
raise ValueError('missing torznab base_url')

3
searx/engines/tubearchivist.py
View File

@@ -75,7 +75,6 @@ Implementations
"""
import typing as t
from urllib.parse import urlencode
from dateutil.parser import parse
from searx.utils import html_to_text, humanize_number
@@ -116,7 +115,7 @@ def absolute_url(relative_url):
return f'{base_url.rstrip("/")}{relative_url}'
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if not base_url:
raise ValueError('tubearchivist engine: base_url is unset')
if not ta_token:

7
searx/engines/tusksearch.py
View File

@@ -13,7 +13,7 @@ from dateutil import parser
from searx.exceptions import SearxEngineAPIException
from searx.network import get
from searx.utils import html_to_text
from searx.utils import gen_useragent, html_to_text
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
@@ -39,7 +39,7 @@ tusk_categ = "web"
api_url = "https://api.tusksearch.com"
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if tusk_categ not in ("web", "images", "videos", "news"):
raise ValueError("invalid search type: %s" % tusk_categ)
@@ -52,7 +52,7 @@ def _obtain_x_sid() -> tuple[str, str]:
The header key is usually called `x-sid-{UUIDv4}`, and the value is
usually a plain UUIDv4 (but a different one than in the header key).
"""
resp = get(f"{api_url}/revcontent/embed.js", headers={"Referer": "https://tusksearch.com/"})
resp = get(f"{api_url}/revcontent/embed.js", headers={"User-Agent": gen_useragent()})
if not resp.ok:
raise SearxEngineAPIException("failed to obtain request x-sid token")
@@ -95,7 +95,6 @@ def request(query: str, params: "OnlineParams") -> None:
# required - we send a random longitude and latitude instead of the actual user location
"x-lon": str(round(random.random() * 90, 4)),
"x-lat": str(round(random.random() * 90, 4)),
"Referer": "https://tusksearch.com/",
}
)

2
searx/engines/uxwing.py
View File

@@ -17,10 +17,12 @@ about = {
categories = ['images', 'icons']
base_url = "https://uxwing.com"
enable_http2 = False
def request(query, params):
params['url'] = f"{base_url}/?s={quote_plus(query)}"
return params
def response(resp):

2
searx/engines/vuhuv.py
View File

@@ -38,7 +38,7 @@ vuhuv_category = "general"
category_map = {"general": 1, "images": 2, "videos": 3}
def setup(_: dict[str, t.Any]) -> bool | None:
def init(_):
if vuhuv_category not in category_map:
raise ValueError("invalid category: %s" % vuhuv_category)

641
searx/engines/wikidata.py
View File

@@ -7,29 +7,24 @@ Some implementations are shared from :ref:`wikipedia engine`.
import typing as t
import os
from hashlib import md5
from urllib.parse import urlencode, unquote
from json import loads
from dateutil.parser import isoparse
from babel.dates import format_datetime, format_date, format_time, get_datetime_format
from searx.enginelib import EngineCache
from searx.data import WIKIDATA_UNITS
from searx.network import post, get
from searx.utils import get_string_replaces_function
from searx.external_urls import area_to_osm_zoom
from searx.utils import searxng_useragent, get_string_replaces_function
from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
from searx.engines.wikipedia import (
fetch_wikimedia_traits,
get_wiki_params,
)
from searx.enginelib.traits import EngineTraits
from searx.wikidata_properties import (
QUERY_TEMPLATE,
WDArticle,
WDAttrList,
WDGeoAttribute,
WDImageAttribute,
WDURLAttribute,
get_attributes,
)
from searx.wikidata import SPARQL_ENDPOINT_URL, SPARQL_EXPLAIN_URL, get_wikidata_headers
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
@@ -52,6 +47,78 @@ display_type = ["infobox"]
one will add a hit to the result list. The first one will show a hit in the
info box. Both values can be set, or one of the two can be set."""
CACHE: EngineCache
"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
seconds."""
# SPARQL
SPARQL_ENDPOINT_URL = "https://query.wikidata.org/sparql"
SPARQL_EXPLAIN_URL = "https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain"
WDPType = dict[str | tuple[str, str], str]
WIKIDATA_PROPERTIES: WDPType = {
"P434": "MusicBrainz",
"P435": "MusicBrainz",
"P436": "MusicBrainz",
"P966": "MusicBrainz",
"P345": "IMDb",
"P2397": "YouTube",
"P1651": "YouTube",
"P2002": "Twitter",
"P2013": "Facebook",
"P2003": "Instagram",
"P4033": "Mastodon",
"P11947": "Lemmy",
"P12622": "PeerTube",
}
# SERVICE wikibase:mwapi : https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual/MWAPI
# SERVICE wikibase:label: https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_Label#Manual_Label_SERVICE
# https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates
# https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format#Data_model
# optimization:
# * https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/query_optimization
# * https://github.com/blazegraph/database/wiki/QueryHints
QUERY_TEMPLATE = """
SELECT ?item ?itemLabel ?itemDescription ?lat ?long %SELECT%
WHERE
{
SERVICE wikibase:mwapi {
bd:serviceParam wikibase:endpoint "www.wikidata.org";
wikibase:api "EntitySearch";
wikibase:limit 1;
mwapi:search "%QUERY%";
mwapi:language "%LANGUAGE%".
?item wikibase:apiOutputItem mwapi:item.
}
hint:Prior hint:runFirst "true".
%WHERE%
SERVICE wikibase:label {
bd:serviceParam wikibase:language "%LANGUAGE%,en".
?item rdfs:label ?itemLabel .
?item schema:description ?itemDescription .
%WIKIBASE_LABELS%
}
}
GROUP BY ?item ?itemLabel ?itemDescription ?lat ?long %GROUP_BY%
"""
# Get the calendar names and the property names
QUERY_PROPERTY_NAMES = """
SELECT ?item ?name
WHERE {
{
SELECT ?item
WHERE { ?item wdt:P279* wd:Q12132 }
} UNION {
VALUES ?item { %ATTRIBUTES% }
}
OPTIONAL { ?item rdfs:label ?name. }
}
"""
# see the property "dummy value" of https://www.wikidata.org/wiki/Q2013 (Wikidata)
# hard coded here to avoid to an additional SPARQL request when the server starts
DUMMY_ENTITY_URLS = set(
@@ -63,13 +130,357 @@ DUMMY_ENTITY_URLS = set(
# https://lists.w3.org/Archives/Public/public-rdf-dawg/2011OctDec/0175.html
sparql_string_escape = get_string_replaces_function(
# fmt: off
{"\t": "\\\t", "\n": "\\\n", "\r": "\\\r", "\b": "\\\b", "\f": "\\\f", "\"": "\\\"", "'": "\\'", "\\": "\\\\"}
{
"\t": "\\\t",
"\n": "\\\n",
"\r": "\\\r",
"\b": "\\\b",
"\f": "\\\f",
"\"": "\\\"",
"\'": "\\\'",
"\\": "\\\\"
}
# fmt: on
)
replace_http_by_https = get_string_replaces_function({"http:": "https:"})
class WDAttribute:
def __init__(self, name: str):
self.name: str = name
def get_select(self):
return "(group_concat(distinct ?{name};separator=', ') as ?{name}s)".replace("{name}", self.name)
def get_label(self, language: str):
return get_label_for_entity(self.name, language)
def get_where(self):
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace("{name}", self.name)
def get_wikibase_label(self) -> str:
return ""
def get_group_by(self) -> str:
return ""
def get_str(self, result: dict[str, t.Any], language: str) -> str | None: # pylint: disable=unused-argument
return result.get(self.name + "s")
def __repr__(self):
return "<" + str(type(self).__name__) + ":" + self.name + ">"
class WDAmountAttribute(WDAttribute):
def get_select(self) -> str:
return "?{name} ?{name}Unit".replace("{name}", self.name)
def get_where(self):
return """ OPTIONAL { ?item p:{name} ?{name}Node .
?{name}Node rdf:type wikibase:BestRank ; ps:{name} ?{name} .
OPTIONAL { ?{name}Node psv:{name}/wikibase:quantityUnit ?{name}Unit. } }""".replace(
'{name}', self.name
)
def get_group_by(self) -> str:
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name)
unit: str | None = result.get(self.name + "Unit")
if unit is not None:
unit = unit.replace("http://www.wikidata.org/entity/", "")
return str(value) + " " + get_label_for_entity(unit, language)
return value
class WDArticle(WDAttribute):
def __init__(self, language: str, kwargs: dict[str, t.Any] | None = None):
super().__init__("wikipedia")
self.language: str = language
self.kwargs: dict[str, t.Any] = kwargs or {}
def get_label(self, language: str):
# language parameter is ignored
return "Wikipedia ({language})".replace("{language}", self.language)
def get_select(self):
return "?article{language} ?articleName{language}".replace("{language}", self.language)
def get_where(self):
return """OPTIONAL { ?article{language} schema:about ?item ;
schema:inLanguage "{language}" ;
schema:isPartOf <https://{language}.wikipedia.org/> ;
schema:name ?articleName{language} . }""".replace(
'{language}', self.language
)
def get_group_by(self):
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
key = "article{language}".replace("{language}", self.language)
return result.get(key)
class WDLabelAttribute(WDAttribute):
def get_select(self):
return "(group_concat(distinct ?{name}Label;separator=', ') as ?{name}Labels)".replace("{name}", self.name)
def get_where(self):
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace("{name}", self.name)
def get_wikibase_label(self) -> str:
return "?{name} rdfs:label ?{name}Label .".replace("{name}", self.name)
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
return result.get(self.name + "Labels")
class WDURLAttribute(WDAttribute):
HTTP_WIKIMEDIA_IMAGE: str = "http://commons.wikimedia.org/wiki/Special:FilePath/"
def __init__(
self,
name: str,
url_id: str | None = None,
url_path_prefix: str | None = None,
kwargs: dict[str, t.Any] | None = None,
):
"""
:param url_id: ID matching one key in ``external_urls.json`` for
converting IDs to full URLs.
:param url_path_prefix: Path prefix if the values are of format
``account@domain``. If provided, value are rewritten to
``https://<domain><url_path_prefix><account>``. For example::
WDURLAttribute('P4033', url_path_prefix='/@')
Adds Property `P4033 <https://www.wikidata.org/wiki/Property:P4033>`_
to the wikidata query. This field might return for example
``libreoffice@fosstodon.org`` and the URL built from this is then:
- account: ``libreoffice``
- domain: ``fosstodon.org``
- result url: https://fosstodon.org/@libreoffice
"""
super().__init__(name)
self.url_id: str | None = url_id
self.url_path_prefix: str | None = url_path_prefix
self.kwargs: dict[str, t.Any] = kwargs or {}
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name + "s")
if not value:
return None
value = value.split(",")[0]
if self.url_id:
url_id = self.url_id
if value.startswith(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE):
value = value[len(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE) :]
url_id = "wikimedia_image"
return get_external_url(url_id, value)
if self.url_path_prefix:
[account, domain] = [x.strip("@ ") for x in value.rsplit("@", 1)]
return f"https://{domain}{self.url_path_prefix}{account}"
return value
class WDGeoAttribute(WDAttribute):
def get_label(self, language: str):
return "OpenStreetMap"
def get_select(self):
return "?{name}Lat ?{name}Long".replace("{name}", self.name)
def get_where(self):
return """OPTIONAL { ?item p:{name}/psv:{name} [
wikibase:geoLatitude ?{name}Lat ;
wikibase:geoLongitude ?{name}Long ] }""".replace(
'{name}', self.name
)
def get_group_by(self):
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
latitude: str | None = result.get(self.name + "Lat")
longitude: str | None = result.get(self.name + "Long")
if latitude and longitude:
return latitude + " " + longitude
return None
def get_geo_url(self, result: dict[str, t.Any], osm_zoom: int = 19) -> str | None:
latitude: str | None = result.get(self.name + "Lat")
longitude: str | None = result.get(self.name + "Long")
if latitude and longitude:
return get_earth_coordinates_url(latitude, longitude, osm_zoom)
return None
class WDImageAttribute(WDURLAttribute):
def __init__(self, name: str, url_id: str | None = None, priority: int = 100):
super().__init__(name, url_id)
self.priority: int = priority
class WDDateAttribute(WDAttribute):
def get_select(self):
return "?{name} ?{name}timePrecision ?{name}timeZone ?{name}timeCalendar".replace("{name}", self.name)
def get_where(self):
# To remove duplicate, add
# FILTER NOT EXISTS { ?item p:{name}/psv:{name}/wikibase:timeValue ?{name}bis FILTER (?{name}bis < ?{name}) }
# this filter is too slow, so the response function ignore duplicate results
# (see the seen_entities variable)
return """OPTIONAL { ?item p:{name}/psv:{name} [
wikibase:timeValue ?{name} ;
wikibase:timePrecision ?{name}timePrecision ;
wikibase:timeTimezone ?{name}timeZone ;
wikibase:timeCalendarModel ?{name}timeCalendar ] . }
hint:Prior hint:rangeSafe true;""".replace(
'{name}', self.name
)
def get_group_by(self):
return self.get_select()
def format_8(self, value: str, locale: str) -> str: # pylint: disable=unused-argument
# precision: less than a year
return value
def format_9(self, value: str, locale: str) -> str:
year = int(value)
# precision: year
if year < 1584:
if year < 0:
return str(year - 1)
return str(year)
timestamp = isoparse(value)
return format_date(timestamp, format="yyyy", locale=locale)
def format_10(self, value: str, locale: str) -> str:
# precision: month
timestamp = isoparse(value)
return format_date(timestamp, format="MMMM y", locale=locale)
def format_11(self, value: str, locale: str) -> str:
# precision: day
timestamp = isoparse(value)
return format_date(timestamp, format="full", locale=locale)
def format_13(self, value: str, locale: str) -> str:
timestamp = isoparse(value)
# precision: minute
return (
get_datetime_format(format, locale=locale)
.replace("'", "")
.replace("{0}", format_time(timestamp, "full", tzinfo=None, locale=locale))
.replace("{1}", format_date(timestamp, "short", locale=locale))
)
def format_14(self, value: str, locale: str) -> str:
# precision: second.
return format_datetime(isoparse(value), format="full", locale=locale)
DATE_FORMAT: dict[str, tuple[str, int]] = {
"0": ("format_8", 1000000000),
"1": ("format_8", 100000000),
"2": ("format_8", 10000000),
"3": ("format_8", 1000000),
"4": ("format_8", 100000),
"5": ("format_8", 10000),
"6": ("format_8", 1000),
"7": ("format_8", 100),
"8": ("format_8", 10),
"9": ("format_9", 1), # year
"10": ("format_10", 1), # month
"11": ("format_11", 0), # day
"12": ("format_13", 0), # hour (not supported by babel, display minute)
"13": ("format_13", 0), # minute
"14": ("format_14", 0), # second
}
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name)
if value == "" or value is None:
return None
_p: str = result.get(self.name + "timePrecision") or "1"
date_format = WDDateAttribute.DATE_FORMAT.get(_p)
if date_format is not None:
format_method = getattr(self, date_format[0])
precision: int = date_format[1]
try:
if precision >= 1:
_t = value.split("-")
if value.startswith("-"):
value = "-" + _t[1]
else:
value = _t[0]
return format_method(value, language)
except Exception: # pylint: disable=broad-except
return value
return value
WDAttrType = (
WDAttribute
| WDAmountAttribute
| WDArticle
| WDLabelAttribute
| WDURLAttribute
| WDGeoAttribute
| WDImageAttribute
| WDDateAttribute
)
WDAttrList = list[WDAttrType]
def get_headers() -> dict[str, str]:
# user agent: https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits
return {
"Accept": "application/sparql-results+json",
"User-Agent": f"wikidata engine - {searxng_useragent()}",
}
def get_label_for_entity(entity_id: str, language: str) -> str:
name = WIKIDATA_PROPERTIES.get(entity_id)
if name is None:
name = WIKIDATA_PROPERTIES.get((entity_id, language))
if name is None:
name = WIKIDATA_PROPERTIES.get((entity_id, language.split("-")[0]))
if name is None:
name = WIKIDATA_PROPERTIES.get((entity_id, "en"))
if name is None:
name = entity_id
return name
def send_wikidata_query(query: str, method: str = "GET", **kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
if method == "GET":
# query will be cached by wikidata
http_response = get(SPARQL_ENDPOINT_URL + "?" + urlencode({"query": query}), headers=get_headers(), **kwargs)
else:
# query won't be cached by wikidata
http_response = post(SPARQL_ENDPOINT_URL, data={"query": query}, headers=get_headers(), **kwargs)
if http_response.status_code != 200:
logger.debug("SPARQL endpoint error %s", http_response.content.decode())
logger.debug("request time %s", str(http_response.elapsed))
http_response.raise_for_status()
return loads(http_response.content.decode())
def request(query: str, params: "OnlineParams") -> None:
attributes: WDAttrList
@@ -80,7 +491,7 @@ def request(query: str, params: "OnlineParams") -> None:
params["method"] = "POST"
params["url"] = SPARQL_ENDPOINT_URL
params["data"] = {"query": query}
params["headers"] = get_wikidata_headers()
params["headers"] = get_headers()
# additional parameters (not a part of OnlineParams)
params["language"] = eng_tag # type: ignore
@@ -173,6 +584,7 @@ def get_results(
for attribute in attributes:
value: str | None = attribute.get_str(attribute_result, language)
if value is not None and value != "":
if isinstance(attribute, (WDURLAttribute, WDArticle)):
# get_select() method : there is group_concat(distinct ...;separator=", ")
# split the value here
@@ -258,15 +670,212 @@ def get_query(query: str, language: str) -> tuple[str, WDAttrList]:
return query, attributes
def get_attributes(language: str):
# pylint: disable=too-many-statements
attributes: WDAttrList = []
def add_value(name: str):
attributes.append(WDAttribute(name))
def add_amount(name: str):
attributes.append(WDAmountAttribute(name))
def add_label(name: str):
attributes.append(WDLabelAttribute(name))
def add_url(name: str, url_id: str | None = None, url_path_prefix: str | None = None, **kwargs: dict[str, t.Any]):
attributes.append(WDURLAttribute(name, url_id, url_path_prefix, kwargs))
def add_image(name: str, url_id: str | None = None, priority: int = 1):
attributes.append(WDImageAttribute(name, url_id, priority))
def add_date(name: str):
attributes.append(WDDateAttribute(name))
# Dates
for p in [
"P571", # inception date
"P576", # dissolution date
"P580", # start date
"P582", # end date
"P569", # date of birth
"P570", # date of death
"P619", # date of spacecraft launch
"P620",
]: # date of spacecraft landing
add_date(p)
for p in [
"P27", # country of citizenship
"P495", # country of origin
"P17", # country
"P159",
]: # headquarters location
add_label(p)
# Places
for p in [
"P36", # capital
"P35", # head of state
"P6", # head of government
"P122", # basic form of government
"P37",
]: # official language
add_label(p)
add_value("P1082") # population
add_amount("P2046") # area
add_amount("P281") # postal code
add_label("P38") # currency
add_amount("P2048") # height (building)
# Media
for p in [
"P400", # platform (videogames, computing)
"P50", # author
"P170", # creator
"P57", # director
"P175", # performer
"P178", # developer
"P162", # producer
"P176", # manufacturer
"P58", # screenwriter
"P272", # production company
"P264", # record label
"P123", # publisher
"P449", # original network
"P750", # distributed by
"P86",
]: # composer
add_label(p)
add_date("P577") # publication date
add_label("P136") # genre (music, film, artistic...)
add_label("P364") # original language
add_value("P212") # ISBN-13
add_value("P957") # ISBN-10
add_label("P275") # copyright license
add_label("P277") # programming language
add_value("P348") # version
add_label("P840") # narrative location
# Languages
add_value("P1098") # number of speakers
add_label("P282") # writing system
add_label("P1018") # language regulatory body
add_value("P218") # language code (ISO 639-1)
# Other
add_label("P169") # ceo
add_label("P112") # founded by
add_label("P1454") # legal form (company, organization)
add_label("P137") # operator (service, facility, ...)
add_label("P1029") # crew members (tripulation)
add_label("P225") # taxon name
add_value("P274") # chemical formula
add_label("P1346") # winner (sports, contests, ...)
add_value("P1120") # number of deaths
add_value("P498") # currency code (ISO 4217)
# URL
kwargs: dict[str, t.Any] = {"official": True}
add_url("P856", **kwargs) # official website
attributes.append(WDArticle(language)) # wikipedia (user language)
if not language.startswith("en"):
attributes.append(WDArticle("en")) # wikipedia (english)
add_url("P1324") # source code repository
add_url("P1581") # blog
add_url("P434", url_id="musicbrainz_artist")
add_url("P435", url_id="musicbrainz_work")
add_url("P436", url_id="musicbrainz_release_group")
add_url("P966", url_id="musicbrainz_label")
add_url("P345", url_id="imdb_id")
add_url("P2397", url_id="youtube_channel")
add_url("P1651", url_id="youtube_video")
add_url("P2002", url_id="twitter_profile")
add_url("P2013", url_id="facebook_profile")
add_url("P2003", url_id="instagram_profile")
# Fediverse
add_url("P4033", url_path_prefix="/@") # Mastodon user
add_url("P11947", url_path_prefix="/c/") # Lemmy community
add_url("P12622", url_path_prefix="/c/") # PeerTube channel
# Map
attributes.append(WDGeoAttribute("P625"))
# Image
add_image("P15", priority=1, url_id="wikimedia_image") # route map
add_image("P242", priority=2, url_id="wikimedia_image") # locator map
add_image("P154", priority=3, url_id="wikimedia_image") # logo
add_image("P18", priority=4, url_id="wikimedia_image") # image
add_image("P41", priority=5, url_id="wikimedia_image") # flag
add_image("P2716", priority=6, url_id="wikimedia_image") # collage
add_image("P2910", priority=7, url_id="wikimedia_image") # icon
return attributes
def debug_explain_wikidata_query(query: str, method: str = "GET"):
if method == "GET":
http_response = get(SPARQL_EXPLAIN_URL + "&" + urlencode({"query": query}), headers=get_wikidata_headers())
http_response = get(SPARQL_EXPLAIN_URL + "&" + urlencode({"query": query}), headers=get_headers())
else:
http_response = post(SPARQL_EXPLAIN_URL, data={"query": query}, headers=get_wikidata_headers())
http_response = post(SPARQL_EXPLAIN_URL, data={"query": query}, headers=get_headers())
http_response.raise_for_status()
return http_response.content
def init(_):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache("wikidata")
# In an environment with competing processes, the initial loading of the
# cache is required only once.
eng_state: str | None = CACHE.get("eng_state")
if not eng_state or not eng_state.startswith("STATE:"):
CACHE.set("eng_state", f"STATE: being initialized by PID {os.getpid()}")
try:
init_wikidata_properties()
except Exception:
CACHE.set("eng_state", f"ERROR: initialization by PID {os.getpid()} failed.")
raise
else:
logger.debug(eng_state)
def init_wikidata_properties():
global WIKIDATA_PROPERTIES # pylint: disable=global-statement
p: WDPType = CACHE.get(key="WIKIDATA_PROPERTIES")
if p:
WIKIDATA_PROPERTIES = p
return
# WIKIDATA_PROPERTIES : add unit symbols
for k, v in WIKIDATA_UNITS.items():
WIKIDATA_PROPERTIES[k] = v["symbol"]
# WIKIDATA_PROPERTIES : add property labels
wikidata_property_names: list[str] = []
for attribute in get_attributes("en"):
if type(attribute) in (WDAttribute, WDAmountAttribute, WDURLAttribute, WDDateAttribute, WDLabelAttribute):
if attribute.name not in WIKIDATA_PROPERTIES:
wikidata_property_names.append("wd:" + attribute.name)
query = QUERY_PROPERTY_NAMES.replace("%ATTRIBUTES%", " ".join(wikidata_property_names))
kwargs: dict[str, t.Any] = {"timeout": 20}
jsonresponse = send_wikidata_query(query, **kwargs)
for result in jsonresponse.get("results", {}).get("bindings", {}):
name_field = result.get("name")
if not name_field:
continue
name = name_field["value"]
lang = name_field["xml:lang"]
entity_id = result["item"]["value"].replace("http://www.wikidata.org/entity/", "")
WIKIDATA_PROPERTIES[(entity_id, lang)] = name.capitalize()
CACHE.set(key="WIKIDATA_PROPERTIES", value=WIKIDATA_PROPERTIES)
def fetch_traits(engine_traits: EngineTraits):
"""Uses languages evaluated from :py:obj:`wikipedia.fetch_wikimedia_traits
<searx.engines.wikipedia.fetch_wikimedia_traits>` and removes

Some files were not shown because too many files have changed in this diff Show More