mirror of
https://github.com/searxng/searxng.git
synced 2026-09-11 16:56:05 +00:00
Compare commits
2 Commits
dependabot
...
00eaf1157d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
00eaf1157d | ||
|
|
f92003b360 |
9
.github/workflows/container.yml
vendored
9
.github/workflows/container.yml
vendored
@@ -62,7 +62,7 @@ jobs:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
@@ -105,9 +105,8 @@ jobs:
|
||||
arch: amd64
|
||||
- runner: ubuntu-26.04-arm
|
||||
arch: arm64
|
||||
# FIXME: https://github.com/searxng/searxng/pull/6655#issuecomment-5550293085
|
||||
# - runner: ubuntu-26.04-arm
|
||||
# arch: armv7
|
||||
- runner: ubuntu-26.04-arm
|
||||
arch: armv7
|
||||
|
||||
steps:
|
||||
- name: Login to GHCR
|
||||
@@ -118,7 +117,7 @@ jobs:
|
||||
password: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
2
.github/workflows/data-update.yml
vendored
2
.github/workflows/data-update.yml
vendored
@@ -31,7 +31,7 @@ jobs:
|
||||
- update_external_bangs.py
|
||||
- update_firefox_version.py
|
||||
- update_engine_traits.py
|
||||
- update_wikidata.py
|
||||
- update_wikidata_units.py
|
||||
- update_engine_descriptions.py
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
/*
|
||||
this file is generated automatically by searxng_extra/update/update_pygments.py
|
||||
using pygments version 2.21.0:
|
||||
using pygments version 2.20.0:
|
||||
|
||||
./manage templates.simple.pygments
|
||||
*/
|
||||
@@ -114,14 +114,14 @@
|
||||
.gd { color: #FF4689 } /* Generic.Deleted */
|
||||
.ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
|
||||
.ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
|
||||
.gr { color: #FF4689 } /* Generic.Error */
|
||||
.gr { color: #F8F8F2 } /* Generic.Error */
|
||||
.gh { color: #F8F8F2 } /* Generic.Heading */
|
||||
.gi { color: #A6E22E } /* Generic.Inserted */
|
||||
.go { color: #66D9EF } /* Generic.Output */
|
||||
.gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
|
||||
.gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
|
||||
.gu { color: #959077 } /* Generic.Subheading */
|
||||
.gt { color: #66D9EF } /* Generic.Traceback */
|
||||
.gt { color: #F8F8F2 } /* Generic.Traceback */
|
||||
.kc { color: #66D9EF } /* Keyword.Constant */
|
||||
.kd { color: #66D9EF } /* Keyword.Declaration */
|
||||
.kn { color: #FF4689 } /* Keyword.Namespace */
|
||||
@@ -132,7 +132,7 @@
|
||||
.m { color: #AE81FF } /* Literal.Number */
|
||||
.s { color: #E6DB74 } /* Literal.String */
|
||||
.na { color: #A6E22E } /* Name.Attribute */
|
||||
.nb { color: #A6E22E } /* Name.Builtin */
|
||||
.nb { color: #F8F8F2 } /* Name.Builtin */
|
||||
.nc { color: #A6E22E } /* Name.Class */
|
||||
.no { color: #66D9EF } /* Name.Constant */
|
||||
.nd { color: #A6E22E } /* Name.Decorator */
|
||||
@@ -166,7 +166,7 @@
|
||||
.sr { color: #E6DB74 } /* Literal.String.Regex */
|
||||
.s1 { color: #E6DB74 } /* Literal.String.Single */
|
||||
.ss { color: #E6DB74 } /* Literal.String.Symbol */
|
||||
.bp { color: #A6E22E } /* Name.Builtin.Pseudo */
|
||||
.bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
|
||||
.fm { color: #A6E22E } /* Name.Function.Magic */
|
||||
.vc { color: #F8F8F2 } /* Name.Variable.Class */
|
||||
.vg { color: #F8F8F2 } /* Name.Variable.Global */
|
||||
|
||||
903
client/simple/package-lock.json
generated
903
client/simple/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -29,21 +29,21 @@
|
||||
"swiped-events": "1.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.5.11",
|
||||
"@types/node": "^26.4.0",
|
||||
"@biomejs/biome": "2.5.7",
|
||||
"@types/node": "^26.2.0",
|
||||
"browserslist": "^4.28.8",
|
||||
"browserslist-to-esbuild": "^2.1.1",
|
||||
"edge.js": "^6.5.1",
|
||||
"less": "^4.9.0",
|
||||
"less": "^4.8.1",
|
||||
"mathjs": "^15.2.0",
|
||||
"sharp": "~0.35.4",
|
||||
"sharp": "~0.35.3",
|
||||
"sort-package-json": "^4.0.0",
|
||||
"stylelint": "^17.14.1",
|
||||
"stylelint-config-standard-less": "^4.1.0",
|
||||
"stylelint-prettier": "^5.0.3",
|
||||
"svgo": "^4.1.0",
|
||||
"svgo": "^4.0.2",
|
||||
"typescript": "~7.0.2",
|
||||
"vite": "^8.2.2",
|
||||
"vite": "^8.2.1",
|
||||
"vite-bundle-analyzer": "^1.3.9"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,9 +58,10 @@ engine is shown. Most of the options have a default value or even are optional.
|
||||
|
||||
# overwrite values from section 'outgoing:'
|
||||
enable_http2: false
|
||||
enable_http3: false
|
||||
retries: 1
|
||||
max_connections: 100
|
||||
max_keepalive_connections: 10
|
||||
keepalive_expiry: 5.0
|
||||
using_tor_proxy: false
|
||||
proxies:
|
||||
http:
|
||||
@@ -162,16 +163,6 @@ engine is shown. Most of the options have a default value or even are optional.
|
||||
``enable_http`` : optional
|
||||
Enable HTTP for this engine (by default only HTTPS is enabled).
|
||||
|
||||
``enable_http3`` : optional
|
||||
Use HTTP/3 (falls back to HTTP/2). Default ``false``.
|
||||
Ignored when a proxy is set.
|
||||
|
||||
.. hint::
|
||||
|
||||
HTTP/3 places demands on the IP infrastructure that are not met in every
|
||||
environment. Enable this option only if you are aware of these requirements
|
||||
and the extent to which they are met.
|
||||
|
||||
``retry_on_http_error`` : optional
|
||||
Retry request on some HTTP status code.
|
||||
|
||||
@@ -188,12 +179,20 @@ engine is shown. Most of the options have a default value or even are optional.
|
||||
Using tor proxy (``true``) or not (``false``) for this engine. The default is
|
||||
taken from ``using_tor_proxy`` of the :ref:`settings outgoing`.
|
||||
|
||||
.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
|
||||
.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
|
||||
|
||||
``max_keepalive_connection#s`` :
|
||||
`Pool limit configuration`_, overwrites value ``pool_maxsize`` from
|
||||
:ref:`settings outgoing` for this engine.
|
||||
|
||||
``max_connections`` :
|
||||
`Pool limit configuration`_, overwrites value ``pool_connections`` from
|
||||
:ref:`settings outgoing` for this engine.
|
||||
|
||||
``keepalive_expiry`` :
|
||||
`Pool limit configuration`_, overwrites value ``keepalive_expiry`` from
|
||||
:ref:`settings outgoing` for this engine.
|
||||
|
||||
|
||||
.. _private engines:
|
||||
|
||||
|
||||
@@ -12,12 +12,20 @@ Communication with search engines.
|
||||
request_timeout: 2.0 # default timeout in seconds, can be override by engine
|
||||
max_request_timeout: 10.0 # the maximum timeout in seconds
|
||||
useragent_suffix: "" # information like an email address to the administrator
|
||||
pool_connections: 100 # Maximum number of concurrent connections (default: 100)
|
||||
enable_http2: true # Enables the use of HTTP2
|
||||
pool_connections: 100 # Maximum number of allowable connections, or null
|
||||
# for no limits. The default is 100.
|
||||
pool_maxsize: 10 # Number of allowable keep-alive connections, or null
|
||||
# to always allow. The default is 10.
|
||||
enable_http2: true # See https://www.python-httpx.org/http2/
|
||||
# uncomment below section if you want to use a custom server certificate
|
||||
# see https://www.python-httpx.org/advanced/#changing-the-verification-defaults
|
||||
# and https://www.python-httpx.org/compatibility/#ssl-configuration
|
||||
# verify: ~/.mitmproxy/mitmproxy-ca-cert.cer
|
||||
#
|
||||
# uncomment below section if you want to use a proxy
|
||||
# uncomment below section if you want to use a proxyq see: SOCKS proxies
|
||||
# https://2.python-requests.org/en/latest/user/advanced/#proxies
|
||||
# are also supported: see
|
||||
# https://2.python-requests.org/en/latest/user/advanced/#socks
|
||||
#
|
||||
# proxies:
|
||||
# all://:
|
||||
@@ -38,26 +46,30 @@ Communication with search engines.
|
||||
timeout to load). Can be override by ``timeout`` in the :ref:`settings engines`.
|
||||
|
||||
``useragent_suffix`` :
|
||||
Suffix to add when an engine's User-Agent is set via searxng_useragent().
|
||||
Contact info here may be useful to avoid an engine blocking you.
|
||||
Suffix to the user-agent SearXNG uses to send requests to others engines. If an
|
||||
engine wish to block you, a contact info here may be useful to avoid that.
|
||||
|
||||
.. _Pool limit configuration: https://curl-cffi.readthedocs.io/en/latest/api.html#sessions
|
||||
.. _Pool limit configuration: https://www.python-httpx.org/advanced/#pool-limit-configuration
|
||||
|
||||
``pool_maxsize``:
|
||||
Number of allowable keep-alive connections, or ``null`` to always allow. The
|
||||
default is 10. See ``max_keepalive_connections`` `Pool limit configuration`_.
|
||||
|
||||
``pool_connections`` :
|
||||
Maximum number of concurrent connections. The default is 100.
|
||||
See ``max_clients`` `Pool limit configuration`_.
|
||||
Maximum number of allowable connections, or ``null`` # for no limits. The
|
||||
default is 100. See ``max_connections`` `Pool limit configuration`_.
|
||||
|
||||
.. _curl_cffi proxies: https://curl-cffi.readthedocs.io/en/latest/quick_start.html
|
||||
``keepalive_expiry`` :
|
||||
Number of seconds to keep a connection in the pool. By default 5.0 seconds.
|
||||
See ``keepalive_expiry`` `Pool limit configuration`_.
|
||||
|
||||
.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying
|
||||
|
||||
``proxies`` :
|
||||
Define one or more proxies you wish to use, see `curl_cffi proxies`_.
|
||||
Define one or more proxies you wish to use, see `httpx proxies`_.
|
||||
If there are more than one proxy for one protocol (http, https),
|
||||
requests to the engines are distributed in a round-robin fashion.
|
||||
|
||||
HTTP, HTTPS, SOCKS4, SOCKS5 and SOCKS5h proxies are supported
|
||||
(``http://``, ``https://``, ``socks4://``, ``socks5://``, ``socks5h://``). You should
|
||||
use ``socks5h://`` when using Tor so hostnames are resolved by the proxy.
|
||||
|
||||
``source_ips`` :
|
||||
If you use multiple network interfaces, define from which IP the requests must
|
||||
be made. Example:
|
||||
@@ -75,15 +87,18 @@ Communication with search engines.
|
||||
different proxy and source ip.
|
||||
|
||||
``enable_http2`` :
|
||||
Enable by default (HTTP/2). Set to ``false`` to force HTTP/1.1.
|
||||
HTTP/3 is opt-in per engine (``enable_http3``).
|
||||
Enable by default. Set to ``false`` to disable HTTP/2.
|
||||
|
||||
.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults
|
||||
.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration
|
||||
|
||||
``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR``
|
||||
HTTPS verification uses the OS's trust store by default.
|
||||
Set a path to use a custom CA file.
|
||||
Allow to specify a path to certificate.
|
||||
see `httpx verification defaults`_.
|
||||
|
||||
In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and
|
||||
``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.
|
||||
see `httpx ssl configuration`_.
|
||||
|
||||
``max_redirects`` :
|
||||
30 by default. Maximum redirect before it is an error.
|
||||
|
||||
@@ -143,7 +143,7 @@ parameters with default value can be redefined for special purposes.
|
||||
data dict ``{}``
|
||||
cookies dict ``{}``
|
||||
verify bool ``True``
|
||||
headers.User-Agent str ``''``
|
||||
headers.User-Agent str a random User-Agent
|
||||
category str current category, like ``'general'``
|
||||
safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict)
|
||||
time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year``
|
||||
@@ -229,8 +229,6 @@ following parameters can be used to specify a search request:
|
||||
max_redirects int maximum redirects, hard limit
|
||||
soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine
|
||||
raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300
|
||||
impersonate str curl_cffi impersonate target (default: chrome, none to disable)
|
||||
curl_options dict Any extra libcurl options for the request
|
||||
=================== =========== ==========================================================================
|
||||
|
||||
|
||||
|
||||
8
docs/dev/engines/online/cara.rst
Normal file
8
docs/dev/engines/online/cara.rst
Normal file
@@ -0,0 +1,8 @@
|
||||
.. _cara engine:
|
||||
|
||||
===========
|
||||
Cara Images
|
||||
===========
|
||||
|
||||
.. automodule:: searx.engines.cara
|
||||
:members:
|
||||
@@ -1,8 +0,0 @@
|
||||
.. _europepmc engine:
|
||||
|
||||
==========
|
||||
Europe PMC
|
||||
==========
|
||||
|
||||
.. automodule:: searx.engines.europepmc
|
||||
:members:
|
||||
@@ -80,8 +80,8 @@ same environment, here are a few examples::
|
||||
# to test one of the update scripts
|
||||
(dev.env)$ searxng_extra/update/update_engine_traits.py --help
|
||||
|
||||
# to test the update of the wikidata units and property names
|
||||
(dev.env)$ searxng_extra/update/update_wikidata.py
|
||||
# to test the update of the wikidata units
|
||||
(dev.env)$ searxng_extra/update/update_wikidata_units.py
|
||||
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
@@ -286,7 +286,7 @@ content becomes smart.
|
||||
files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst```
|
||||
pull request :pull:`4` ``:pull:`4```
|
||||
patch :patch:`af2cae6` ``:patch:`af2cae6```
|
||||
PyPi package :pypi:`curl_cffi` ``:pypi:`curl_cffi```
|
||||
PyPi package :pypi:`httpx` ``:pypi:`httpx```
|
||||
manual page man :man:`bash` ``:man:`bash```
|
||||
intersphinx_
|
||||
--------------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -90,10 +90,10 @@ Scripts to update static data in :origin:`searx/data/`
|
||||
:members:
|
||||
|
||||
|
||||
``update_wikidata.py``
|
||||
``update_wikidata_units.py``
|
||||
============================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_wikidata.py>`
|
||||
:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_wikidata
|
||||
.. automodule:: searxng_extra.update.update_wikidata_units
|
||||
:members:
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
mock==5.2.0
|
||||
nose2[coverage_plugin]==0.16.0
|
||||
cov-core==1.15.0
|
||||
black==25.9.0
|
||||
pylint==4.0.8
|
||||
black==26.5.1
|
||||
pylint==4.0.7
|
||||
splinter==0.21.0
|
||||
selenium==4.48.0
|
||||
selenium==4.47.0
|
||||
Sphinx==8.2.3;python_version <= "3.11"
|
||||
Sphinx==9.1.0; python_version > "3.11"
|
||||
sphinx-issues==6.0.0
|
||||
@@ -18,11 +18,11 @@ myst-parser==5.0.0
|
||||
linuxdoc==20260504
|
||||
aiounittest==1.5.0
|
||||
yamllint==1.38.0
|
||||
wlc==2.2.0
|
||||
wlc==2.1.1
|
||||
coloredlogs==15.0.1
|
||||
docutils>=0.21.2;python_version <= "3.11"
|
||||
docutils>=0.22.4; python_version > "3.11"
|
||||
parameterized==0.9.0
|
||||
granian[reload]==2.8.2
|
||||
basedpyright==1.40.0
|
||||
granian[reload]==2.8.1
|
||||
basedpyright==1.39.9
|
||||
types-lxml==2026.2.16
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
granian==2.8.2
|
||||
granian[pname]==2.8.2
|
||||
granian==2.8.1
|
||||
granian[pname]==2.8.1
|
||||
|
||||
@@ -3,15 +3,17 @@ babel==2.18.0
|
||||
flask-babel==4.0.0
|
||||
flask==3.1.3
|
||||
jinja2==3.1.6
|
||||
lxml==6.1.3
|
||||
pygments==2.21.0
|
||||
lxml==6.1.1
|
||||
pygments==2.20.0
|
||||
python-dateutil==2.9.0.post0
|
||||
pyyaml==6.0.3
|
||||
curl_cffi==0.16.3
|
||||
httpx[http2]==0.28.1
|
||||
httpx-socks[asyncio]==0.13.1
|
||||
sniffio==1.3.1
|
||||
valkey==6.1.1
|
||||
markdown-it-py==4.2.0
|
||||
msgspec==0.21.1
|
||||
typer==0.27.2
|
||||
typer==0.27.1
|
||||
isodate==0.7.2
|
||||
whitenoise==6.12.0
|
||||
typing-extensions==4.16.0
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Implementation of the :py:obj:`preference <searx.preference>` settings."""
|
||||
|
||||
# pylint: disable = too-few-public-methods
|
||||
|
||||
import typing as t
|
||||
|
||||
@@ -38,7 +38,6 @@ area:
|
||||
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["AnswererInfo", "Answerer", "AnswerStorage"]
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ from dataclasses import dataclass
|
||||
from searx.utils import load_module
|
||||
from searx.result_types.answer import BaseAnswer
|
||||
|
||||
|
||||
_default = pathlib.Path(__file__).parent
|
||||
log: logging.Logger = logging.getLogger("searx.answerers")
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ from urllib.parse import urlencode
|
||||
|
||||
import lxml.etree
|
||||
import lxml.html
|
||||
from curl_cffi.requests.exceptions import RequestException
|
||||
from httpx import HTTPError
|
||||
|
||||
from searx import settings
|
||||
from searx.engines import (
|
||||
@@ -63,7 +63,7 @@ def bing(query: str, _sxng_locale: str) -> list[str]:
|
||||
base_url = "https://www.bing.com/AS/Suggestions?"
|
||||
# cvid has to be a 32 character long string consisting of numbers and uppsercase characters
|
||||
cvid = ''.join(random.choices(string.ascii_uppercase + string.digits, k=32))
|
||||
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}), enable_http3=True)
|
||||
response = get(base_url + urlencode({'qry': query, 'csr': 1, 'cvid': cvid}))
|
||||
results: list[str] = []
|
||||
|
||||
if response.ok:
|
||||
@@ -83,7 +83,7 @@ def brave(query: str, _sxng_locale: str) -> list[str]:
|
||||
url = 'https://search.brave.com/api/suggest?'
|
||||
url += urlencode({'q': query})
|
||||
country = 'all'
|
||||
kwargs = {'cookies': {'country': country}, 'enable_http3': True}
|
||||
kwargs = {'cookies': {'country': country}}
|
||||
resp = get(url, **kwargs)
|
||||
results: list[str] = []
|
||||
|
||||
@@ -127,17 +127,18 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
|
||||
|
||||
|
||||
def google_complete(query: str, sxng_locale: str) -> list[str]:
|
||||
"""Autocomplete from Google. Supports Google's languages
|
||||
"""Autocomplete from Google. Supports Google's languages and subdomains
|
||||
(:py:obj:`searx.engines.google.get_google_info`) by using the async REST
|
||||
API::
|
||||
|
||||
https://www.google.com/complete/search?{args}
|
||||
https://{subdomain}/complete/search?{args}
|
||||
|
||||
"""
|
||||
|
||||
data = ENGINE_TRAITS.get("google") or {}
|
||||
traits = EngineTraits(**data)
|
||||
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
|
||||
url = 'https://{subdomain}/complete/search?{args}'
|
||||
args = urlencode(
|
||||
{
|
||||
'q': query,
|
||||
@@ -147,7 +148,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
|
||||
)
|
||||
results: list[str] = []
|
||||
|
||||
resp = get('https://www.google.com/complete/search?' + args, enable_http3=True)
|
||||
resp = get(url.format(subdomain=google_info['subdomain'], args=args))
|
||||
if resp and resp.ok:
|
||||
json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
|
||||
data = json.loads(json_txt)
|
||||
@@ -418,5 +419,5 @@ def search_autocomplete(backend_name: str, query: str, sxng_locale: str) -> list
|
||||
return []
|
||||
try:
|
||||
return backend(query, sxng_locale)
|
||||
except (RequestException, SearxEngineResponseException):
|
||||
except (HTTPError, SearxEngineResponseException):
|
||||
return []
|
||||
|
||||
@@ -5,7 +5,6 @@ Implementations used for bot detection.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["init", "dump_request", "get_network", "too_many_requests", "ProxyFix"]
|
||||
|
||||
|
||||
|
||||
@@ -182,7 +182,7 @@ class Config:
|
||||
if default is UNSET:
|
||||
raise KeyError(name)
|
||||
return default
|
||||
(modulename, name) = str(fqn).rsplit('.', 1)
|
||||
modulename, name = str(fqn).rsplit('.', 1)
|
||||
m = __import__(modulename, {}, {}, [name], 0)
|
||||
return getattr(m, name)
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ Accept_ header ..
|
||||
|
||||
"""
|
||||
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
|
||||
@@ -14,7 +14,6 @@ bot if the Accept-Encoding_ header ..
|
||||
|
||||
"""
|
||||
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
|
||||
@@ -11,7 +11,6 @@ if the Accept-Language_ header is unset.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
|
||||
@@ -11,7 +11,6 @@ the Connection_ header is set to ``close``.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
|
||||
@@ -20,6 +20,7 @@ Metadata`_. A request is filtered out in case of:
|
||||
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ the User-Agent_ header is unset or matches the regular expression
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import re
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
@@ -25,7 +24,6 @@ import flask
|
||||
from . import config
|
||||
from ._helpers import too_many_requests
|
||||
|
||||
|
||||
USER_AGENT = (
|
||||
r'('
|
||||
+ r'unknown'
|
||||
|
||||
@@ -55,7 +55,6 @@ from ._helpers import (
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
logger = logger.getChild('ip_limit')
|
||||
|
||||
BURST_WINDOW = 20
|
||||
|
||||
@@ -23,6 +23,7 @@ The ``ip_lists`` method implements :py:obj:`block-list <block_ip>` and
|
||||
]
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Implementation of a middleware to determine the real IP of an HTTP request
|
||||
(:py:obj:`flask.request.remote_addr`) behind a proxy chain."""
|
||||
|
||||
# pylint: disable=too-many-branches
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Providing a Valkey database for the botdetection methods."""
|
||||
|
||||
|
||||
import valkey
|
||||
|
||||
__all__ = ["set_valkey_client", "get_valkey_client"]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Implementations needed for a branding of SearXNG."""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
|
||||
# Struct fields aren't discovered in Python 3.14
|
||||
|
||||
@@ -465,7 +465,7 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
|
||||
|
||||
# Check if value is expired. It's possible that it's expired but has not
|
||||
# yet been automatically deleted by the periodic maintenance
|
||||
(value, expire) = row
|
||||
value, expire = row
|
||||
now = time.time()
|
||||
if expire < now:
|
||||
# The record is deleted during the maintenance interval. Deleting
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import warnings
|
||||
|
||||
|
||||
# limiter backward compatibility
|
||||
# ------------------------------
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
make data.all
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
|
||||
__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
|
||||
@@ -32,13 +33,6 @@ class WikiDataUnitType(t.TypedDict):
|
||||
to_si_factor: float
|
||||
|
||||
|
||||
WikiDataPropertyNameType = str | dict[str, str]
|
||||
"""Name of a Wikidata property. Can be either the plain name or a dictionary of
|
||||
language code to property name, e.g. ``{"en": "Date of birth"}``."""
|
||||
WikiDataPropertiesType = dict[str, WikiDataPropertyNameType]
|
||||
"""Dictionary from wikidata property ID to property name."""
|
||||
|
||||
|
||||
class LocalesType(t.TypedDict):
|
||||
"""Data structure of an item in ``locales.json``"""
|
||||
|
||||
@@ -48,7 +42,6 @@ class LocalesType(t.TypedDict):
|
||||
|
||||
USER_AGENTS: UserAgentType
|
||||
WIKIDATA_UNITS: dict[str, WikiDataUnitType]
|
||||
WIKIDATA_PROPERTIES: WikiDataPropertiesType
|
||||
TRACKER_PATTERNS: TrackerPatternsDB
|
||||
LOCALES: LocalesType
|
||||
CURRENCIES: CurrenciesDB
|
||||
@@ -60,12 +53,11 @@ ENGINE_DESCRIPTIONS: dict[str, dict[str, t.Any]]
|
||||
ENGINE_TRAITS: dict[str, dict[str, t.Any]]
|
||||
|
||||
|
||||
lazy_globals: dict[str, t.Any] = {
|
||||
lazy_globals = {
|
||||
"CURRENCIES": CurrenciesDB(),
|
||||
"USER_AGENTS": None,
|
||||
"EXTERNAL_URLS": None,
|
||||
"WIKIDATA_UNITS": None,
|
||||
"WIKIDATA_PROPERTIES": None,
|
||||
"EXTERNAL_BANGS": None,
|
||||
"OSM_KEYS_TAGS": None,
|
||||
"ENGINE_DESCRIPTIONS": None,
|
||||
@@ -78,7 +70,6 @@ data_json_files = {
|
||||
"USER_AGENTS": "useragents.json",
|
||||
"EXTERNAL_URLS": "external_urls.json",
|
||||
"WIKIDATA_UNITS": "wikidata_units.json",
|
||||
"WIKIDATA_PROPERTIES": "wikidata_properties.json",
|
||||
"EXTERNAL_BANGS": "external_bangs.json",
|
||||
"OSM_KEYS_TAGS": "osm_keys_tags.json",
|
||||
"ENGINE_DESCRIPTIONS": "engine_descriptions.json",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Simple implementation to store TrackerPatterns data in a SQL database."""
|
||||
|
||||
# pylint: disable=too-many-branches
|
||||
|
||||
import typing as t
|
||||
@@ -10,7 +11,7 @@ import re
|
||||
from collections.abc import Iterator
|
||||
from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
|
||||
|
||||
from curl_cffi.requests.exceptions import RequestException
|
||||
from httpx import HTTPError
|
||||
|
||||
from searx.data.core import get_cache, log
|
||||
from searx.network import get as http_get
|
||||
@@ -28,11 +29,11 @@ class TrackerPatternsDB:
|
||||
|
||||
ctx_name = "data_tracker_patterns"
|
||||
|
||||
# ClearURL rule lists, the first one that responds HTTP 200 is used
|
||||
CLEAR_LIST_URL = [
|
||||
"https://cdn.jsdelivr.net/gh/clearurls/rules@refs/heads/gh-pages/data.minify.json",
|
||||
"https://rules2.clearurls.xyz/data.minify.json",
|
||||
# ClearURL rule lists, the first one that responds HTTP 200 is used
|
||||
"https://rules1.clearurls.xyz/data.minify.json",
|
||||
"https://rules2.clearurls.xyz/data.minify.json",
|
||||
"https://raw.githubusercontent.com/ClearURLs/Rules/refs/heads/master/data.min.json",
|
||||
]
|
||||
|
||||
class Fields:
|
||||
@@ -87,8 +88,8 @@ class TrackerPatternsDB:
|
||||
try:
|
||||
resp = http_get(url, timeout=3)
|
||||
|
||||
except RequestException as exc:
|
||||
log.warning("TRACKER_PATTERNS: RequestException while fetching %s: %s", url, exc)
|
||||
except HTTPError as exc:
|
||||
log.warning("TRACKER_PATTERNS: HTTPError (%s) occured while fetching %s", url, exc)
|
||||
continue
|
||||
|
||||
if resp.status_code != 200:
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
],
|
||||
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
|
||||
"versions": [
|
||||
"154.0",
|
||||
"153.0"
|
||||
"153.0",
|
||||
"152.0"
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3474,6 +3474,11 @@
|
||||
"symbol": "mm⁻²",
|
||||
"to_si_factor": 1e-06
|
||||
},
|
||||
"Q136039973": {
|
||||
"si_name": "Q6137407",
|
||||
"symbol": "FPS",
|
||||
"to_si_factor": 1.0
|
||||
},
|
||||
"Q1361854": {
|
||||
"si_name": "Q11570",
|
||||
"symbol": "dwt",
|
||||
@@ -5249,11 +5254,6 @@
|
||||
"symbol": "μA",
|
||||
"to_si_factor": 1e-06
|
||||
},
|
||||
"Q31274648": {
|
||||
"si_name": "Q6137407",
|
||||
"symbol": "FPS",
|
||||
"to_si_factor": 1.0
|
||||
},
|
||||
"Q3186734": {
|
||||
"si_name": "Q3186734",
|
||||
"symbol": "J/(m³ K)",
|
||||
|
||||
@@ -305,7 +305,7 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
|
||||
region: str = ""
|
||||
"""For an engine, when there is ``region: ...`` in the YAML settings the engine
|
||||
does support only this one region:
|
||||
does support only this one region::
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
@@ -317,9 +317,6 @@ class Engine(abc.ABC): # pylint: disable=too-few-public-methods
|
||||
enable_http: bool
|
||||
"""Enable HTTP (by default only HTTPS is enabled)."""
|
||||
|
||||
enable_http3: bool = False
|
||||
"""Enables the use of HTTP/3 if available"""
|
||||
|
||||
shortcut: str
|
||||
"""Code used to execute bang requests (``!foo``)"""
|
||||
|
||||
|
||||
@@ -6,8 +6,7 @@ from urllib.parse import urlencode
|
||||
from datetime import datetime
|
||||
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import html_to_text
|
||||
from searx.utils import html_to_text, get_embeded_stream_url
|
||||
|
||||
about = {
|
||||
"website": "https://tv.360kan.com/",
|
||||
@@ -30,12 +29,12 @@ def request(query, params):
|
||||
return params
|
||||
|
||||
|
||||
def response(resp) -> EngineResults:
|
||||
def response(resp):
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise SearxEngineAPIException(f"Invalid response: {e}") from e
|
||||
res = EngineResults()
|
||||
results = []
|
||||
|
||||
if "data" not in data or "result" not in data["data"]:
|
||||
raise SearxEngineAPIException("Invalid response")
|
||||
@@ -51,15 +50,16 @@ def response(resp) -> EngineResults:
|
||||
except (ValueError, TypeError):
|
||||
published_date = None
|
||||
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
url=entry["play_url"],
|
||||
title=html_to_text(entry["title"]),
|
||||
content=html_to_text(entry["description"]),
|
||||
template='videos.html',
|
||||
publishedDate=published_date,
|
||||
thumbnail=entry["cover_img"],
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
'url': entry["play_url"],
|
||||
'title': html_to_text(entry["title"]),
|
||||
'content': html_to_text(entry["description"]),
|
||||
'template': 'videos.html',
|
||||
'publishedDate': published_date,
|
||||
'thumbnail': entry["cover_img"],
|
||||
"iframe_src": get_embeded_stream_url(entry["play_url"]),
|
||||
}
|
||||
)
|
||||
|
||||
return res
|
||||
return results
|
||||
|
||||
@@ -26,7 +26,6 @@ categories: list[str]
|
||||
disabled: bool
|
||||
display_error_messages: bool
|
||||
enable_http: bool
|
||||
enable_http3: bool
|
||||
engine_type: str
|
||||
inactive: bool
|
||||
max_page: int
|
||||
|
||||
@@ -187,9 +187,8 @@ def set_loggers(engine: "Engine|types.ModuleType", engine_name: str):
|
||||
def update_engine_attributes(engine: "Engine | types.ModuleType", engine_data: dict[str, t.Any]):
|
||||
# pylint: disable=too-many-branches
|
||||
|
||||
# set / update engine attributes from engine_data
|
||||
# set engine attributes from engine_data
|
||||
kvargs: dict[str, t.Any]
|
||||
engine.about = getattr(engine, "about", EngineAbout())
|
||||
if isinstance(engine.about, EngineAbout):
|
||||
kvargs = {**msgspec.to_builtins(engine.about), **engine_data.get("about", {})}
|
||||
else:
|
||||
|
||||
@@ -35,7 +35,7 @@ Implementation
|
||||
==============
|
||||
|
||||
"""
|
||||
import typing as t
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.parse import urlencode
|
||||
|
||||
@@ -85,7 +85,7 @@ Additional subcategories:
|
||||
# Do we need support for "free_collection" and "include_stock_enterprise"?
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if not categories:
|
||||
raise ValueError("adobe_stock engine: categories is unset")
|
||||
|
||||
@@ -100,9 +100,9 @@ def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
raise ValueError("adobe_stock engine: adobe_content_types is unset")
|
||||
|
||||
if isinstance(adobe_content_types, list):
|
||||
for content_type in adobe_content_types:
|
||||
if content_type not in ADOBE_VALID_TYPES:
|
||||
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % content_type)
|
||||
for t in adobe_content_types:
|
||||
if t not in ADOBE_VALID_TYPES:
|
||||
raise ValueError("adobe_stock engine: adobe_content_types: '%s' is invalid" % t)
|
||||
else:
|
||||
raise ValueError(
|
||||
"adobe_stock engine: adobe_content_types must be a list of strings not %s" % type(adobe_content_types)
|
||||
|
||||
@@ -25,6 +25,7 @@ To use this engine, add an entry similar to the following to your engine list in
|
||||
https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
|
||||
@@ -49,9 +49,6 @@ CACHE: EngineCache
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
if baidu_category not in ('general', 'images', 'it'):
|
||||
raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
|
||||
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
@@ -68,6 +65,11 @@ def get_image_cookies(headers: dict[str, str]) -> dict[str, str]:
|
||||
return cookies
|
||||
|
||||
|
||||
def init(_):
|
||||
if baidu_category not in ('general', 'images', 'it'):
|
||||
raise SearxEngineAPIException(f"Unsupported category: {baidu_category}")
|
||||
|
||||
|
||||
def request(query, params):
|
||||
page_num = params["pageno"]
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""BASE (Scholar publications)"""
|
||||
|
||||
from datetime import datetime
|
||||
import re
|
||||
|
||||
|
||||
@@ -40,7 +40,6 @@ about: dict[str, t.Any] = {
|
||||
# engine dependent config
|
||||
categories = ["general", "web"]
|
||||
safesearch = True
|
||||
enable_http3 = True
|
||||
_safesearch_map: dict[int, str] = {
|
||||
0: "off",
|
||||
1: "moderate",
|
||||
@@ -72,21 +71,43 @@ def get_locale_params(engine_region: str | None) -> dict[str, str] | None:
|
||||
return {"mkt": engine_region}
|
||||
|
||||
|
||||
def override_accept_language(params: "OnlineParams", engine_region: str | None) -> None:
|
||||
"""Override the ``Accept-Language`` header.
|
||||
|
||||
The default header built by :py:class:`~searx.search.processors.online.OnlineProcessor`
|
||||
appends ``en;q=0.3`` as a fallback language::
|
||||
|
||||
Accept-Language: de,de-DE;q=0.7,en;q=0.3
|
||||
|
||||
Bing seems to better select the results locale based on the
|
||||
``Accept-Language`` value header.
|
||||
|
||||
This function is shared with :py:mod:`searx.engines.bing_images`,
|
||||
:py:mod:`searx.engines.bing_news`, and :py:mod:`searx.engines.bing_videos`.
|
||||
"""
|
||||
|
||||
if not engine_region or engine_region == "clear":
|
||||
return
|
||||
|
||||
lang = engine_region.split("-")[0]
|
||||
params["headers"]["Accept-Language"] = f"{engine_region},{lang};q=0.9"
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams"):
|
||||
"""Assemble a Bing-Web request."""
|
||||
|
||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||
|
||||
override_accept_language(params, engine_region)
|
||||
|
||||
query_params: dict[str, str | int] = {
|
||||
"q": query,
|
||||
"adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
|
||||
}
|
||||
|
||||
if engine_region and engine_region != "clear":
|
||||
lang, _, cc = engine_region.partition("-")
|
||||
query_params["setlang"] = lang
|
||||
if cc and cc not in ("us", "cn", "ru"): # bing just sends junk for these
|
||||
query_params["cc"] = cc
|
||||
locale_params = get_locale_params(engine_region)
|
||||
if locale_params:
|
||||
query_params.update(locale_params)
|
||||
|
||||
params["url"] = f"{base_url}/search?{urlencode(query_params)}"
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from lxml import html
|
||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||
fetch_traits,
|
||||
get_locale_params,
|
||||
override_accept_language,
|
||||
)
|
||||
|
||||
# about
|
||||
@@ -24,7 +25,6 @@ about = {
|
||||
# engine dependent config
|
||||
categories = ["images", "web"]
|
||||
paging = True
|
||||
enable_http3 = True
|
||||
safesearch = True
|
||||
time_range_support = True
|
||||
time_map = {
|
||||
@@ -43,6 +43,8 @@ def request(query, params):
|
||||
|
||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||
|
||||
override_accept_language(params, engine_region)
|
||||
|
||||
# build URL query
|
||||
# - example: https://www.bing.com/images/async?q=foo&async=1&first=1&count=35
|
||||
query_params = {
|
||||
|
||||
@@ -12,7 +12,10 @@ from urllib.parse import urlencode
|
||||
from lxml import html
|
||||
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
from searx.engines.bing import get_locale_params
|
||||
from searx.engines.bing import (
|
||||
get_locale_params,
|
||||
override_accept_language,
|
||||
)
|
||||
from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
|
||||
|
||||
# about
|
||||
@@ -30,7 +33,6 @@ categories = ["news"]
|
||||
paging = True
|
||||
"""If go through the pages and there are actually no new results for another
|
||||
page, then bing returns the results from the last page again."""
|
||||
enable_http3 = True
|
||||
|
||||
time_range_support = True
|
||||
time_map = {
|
||||
@@ -51,6 +53,8 @@ def request(query, params):
|
||||
|
||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||
|
||||
override_accept_language(params, engine_region)
|
||||
|
||||
# build URL query
|
||||
# - example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
|
||||
page = int(params.get("pageno", 1)) - 1
|
||||
|
||||
@@ -9,6 +9,7 @@ from lxml import html
|
||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||
fetch_traits,
|
||||
get_locale_params,
|
||||
override_accept_language,
|
||||
)
|
||||
from searx.engines.bing_images import time_map
|
||||
from searx.utils import eval_xpath, eval_xpath_getindex
|
||||
@@ -25,7 +26,6 @@ about = {
|
||||
# engine dependent config
|
||||
categories = ["videos", "web"]
|
||||
paging = True
|
||||
enable_http3 = True
|
||||
safesearch = True
|
||||
time_range_support = True
|
||||
|
||||
@@ -38,6 +38,8 @@ def request(query, params):
|
||||
|
||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||
|
||||
override_accept_language(params, engine_region)
|
||||
|
||||
# build URL query
|
||||
# - example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
|
||||
query_params = {
|
||||
|
||||
@@ -45,7 +45,7 @@ CACHE_SESSION_ID_KEY = "session_id_key"
|
||||
KEYWORD_RE = re.compile(r"\[\/?Keyword\]")
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
def init(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_name=engine_settings["name"])
|
||||
return True
|
||||
|
||||
@@ -135,7 +135,7 @@ from searx.utils import (
|
||||
eval_xpath_getindex,
|
||||
eval_xpath_list,
|
||||
extract_text,
|
||||
get_embedded_stream_url,
|
||||
get_embeded_stream_url,
|
||||
js_obj_str_to_json_str,
|
||||
js_obj_str_to_python,
|
||||
)
|
||||
@@ -151,7 +151,6 @@ about = {
|
||||
|
||||
base_url = "https://search.brave.com/"
|
||||
categories = []
|
||||
enable_http3 = True
|
||||
brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = "search"
|
||||
"""Brave supports common web-search, videos, images, news, and goggles search.
|
||||
|
||||
@@ -248,13 +247,13 @@ def extract_json_data(text: str) -> dict[str, t.Any]:
|
||||
# node_ids: [0, 19],
|
||||
# data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]
|
||||
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
# form: null,
|
||||
# error: null
|
||||
# });
|
||||
text = text[text.index("<script") : text.index("</script")]
|
||||
if not text:
|
||||
raise ValueError("can't find JS/JSON data in the given text")
|
||||
start = text.index("data: [{")
|
||||
newline = text.index("\n", start)
|
||||
end = text.rindex("}}]", start, newline)
|
||||
js_obj_str = "{" + text[start:end] + "}}]}"
|
||||
end = text.rindex("}}]")
|
||||
js_obj_str = text[start:end]
|
||||
js_obj_str = "{" + js_obj_str + "}}]}"
|
||||
# js_obj_str = js_obj_str.replace("\xa0", "") # remove ASCII for
|
||||
# js_obj_str = js_obj_str.replace(r"\u003C", "<").replace(r"\u003c", "<") # fix broken HTML tags in strings
|
||||
json_str = js_obj_str_to_json_str(js_obj_str)
|
||||
@@ -339,7 +338,7 @@ def _parse_search(resp: SXNG_Response) -> EngineResults:
|
||||
if len(video_tag):
|
||||
# In my tests a video tag in the WEB search was most often not a
|
||||
# video, except the ones from youtube ..
|
||||
iframe_src = get_embedded_stream_url(url)
|
||||
iframe_src = get_embeded_stream_url(url)
|
||||
if iframe_src:
|
||||
item["iframe_src"] = iframe_src
|
||||
item["template"] = "videos.html"
|
||||
@@ -354,14 +353,14 @@ def _parse_news(resp: SXNG_Response) -> EngineResults:
|
||||
res = EngineResults()
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
for result in eval_xpath_list(dom, "//div[@data-type='news']"):
|
||||
url = eval_xpath_getindex(result, ".//a/@href", 0, default=None)
|
||||
for result in eval_xpath_list(dom, "//div[contains(@class, 'results')]//div[@data-type='news']"):
|
||||
url = eval_xpath_getindex(result, ".//a[contains(@class, 'result-header')]/@href", 0, default=None)
|
||||
if url is None:
|
||||
continue
|
||||
|
||||
title = eval_xpath_list(result, ".//div[contains(@class, 'title')]")
|
||||
content = eval_xpath_list(result, ".//div[contains(@class, 'description')]")
|
||||
thumbnail = eval_xpath_getindex(result, ".//a[contains(@class, 'thumbnail')]//img/@src", 0, default="")
|
||||
title = eval_xpath_list(result, ".//span[contains(@class, 'snippet-title')]")
|
||||
content = eval_xpath_list(result, ".//p[contains(@class, 'desc')]")
|
||||
thumbnail = eval_xpath_getindex(result, ".//div[contains(@class, 'image-wrapper')]//img/@src", 0, default="")
|
||||
|
||||
item = res.types.LegacyResult(
|
||||
template="default.html",
|
||||
@@ -407,6 +406,9 @@ def _parse_videos(json_resp: dict[str, t.Any]) -> EngineResults:
|
||||
)
|
||||
if result["thumbnail"] is not None:
|
||||
item["thumbnail"] = result["thumbnail"]["src"]
|
||||
iframe_src = get_embeded_stream_url(result["url"])
|
||||
if iframe_src:
|
||||
item["iframe_src"] = iframe_src
|
||||
|
||||
res.add(item)
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ if t.TYPE_CHECKING:
|
||||
about = {
|
||||
"website": "https://api.search.brave.com/",
|
||||
"wikidata_id": None,
|
||||
"official_api_documentation": "https://api-dashboard.search.brave.com/api-reference/web/search/get",
|
||||
"official_api_documentation": "https://api-dashboard.search.brave.com/documentation",
|
||||
"use_official_api": True,
|
||||
"require_api_key": True,
|
||||
"results": "JSON",
|
||||
@@ -63,10 +63,8 @@ base_url = "https://api.search.brave.com/res/v1/web/search"
|
||||
time_range_map = {"day": "past_day", "week": "past_week", "month": "past_month", "year": "past_year"}
|
||||
"""Mapping of SearXNG time ranges to Brave API time ranges."""
|
||||
|
||||
max_page = 10
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
"""Initialize the engine."""
|
||||
if not api_key:
|
||||
raise SearxEngineAPIException("No API key provided")
|
||||
@@ -77,7 +75,7 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
search_args: dict[str, str | int | None] = {
|
||||
"q": query,
|
||||
"count": results_per_page,
|
||||
"offset": params["pageno"] - 1,
|
||||
"offset": (params["pageno"] - 1) * results_per_page,
|
||||
"text_decorations": False,
|
||||
}
|
||||
|
||||
@@ -91,7 +89,6 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
params["url"] = f"{base_url}?{urlencode(search_args)}"
|
||||
params["headers"]["X-Subscription-Token"] = api_key
|
||||
params["headers"]["Accept"] = "application/json"
|
||||
|
||||
|
||||
def _extract_published_date(published_date_raw: str):
|
||||
|
||||
85
searx/engines/cara.py
Normal file
85
searx/engines/cara.py
Normal file
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# pylint: disable=invalid-name
|
||||
"""Cara_ is a social media and portfolio-sharing platform for artists and art
|
||||
enthusiasts.
|
||||
|
||||
With the widespread use of generative AI, Cara_ decided to build a place that
|
||||
filters out gen AI images so that people searching for authentic creatives and
|
||||
images can do so easily.
|
||||
|
||||
.. _Cara: https://cara.app/about
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import typing as t
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
|
||||
about = {
|
||||
"website": "https://cara.app",
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
base_url = "https://cara.app"
|
||||
images_url = "https://images.cara.app"
|
||||
|
||||
categories = ["images"]
|
||||
paging = True
|
||||
results_per_page = 24
|
||||
|
||||
# if using HTTP2, we get blocked immediately
|
||||
enable_http2 = False
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
args = {
|
||||
"q": query,
|
||||
"sortBy": "Top",
|
||||
"take": results_per_page,
|
||||
"skip": (params["pageno"] - 1) * results_per_page,
|
||||
}
|
||||
params["url"] = f"{base_url}/api/search/portfolio-posts?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
res = EngineResults()
|
||||
json_data: list[dict[str, t.Any]] = resp.json()
|
||||
|
||||
for result in json_data:
|
||||
thumbnail, img = None, None
|
||||
|
||||
i: dict[str, str]
|
||||
for i in result["images"]:
|
||||
if thumbnail is None or i["isCoverImg"]:
|
||||
thumbnail = i
|
||||
|
||||
if img is None or not i["isCoverImg"]:
|
||||
img = i
|
||||
|
||||
if not thumbnail or not img:
|
||||
continue
|
||||
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
{
|
||||
"template": "images.html",
|
||||
"url": f"{base_url}/post/{result['id']}",
|
||||
"thumbnail_src": f"{images_url}/{thumbnail['src']}?height=256",
|
||||
"img_src": f"{images_url}/{img['src']}",
|
||||
"title": result["title"],
|
||||
"content": result["content"],
|
||||
"author": result["name"],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
@@ -41,7 +41,7 @@ search_index = "cw22"
|
||||
<https://www.chatnoir.eu/docs/api-general>`_ for a full list."""
|
||||
|
||||
|
||||
def _obtain_api_key() -> tuple[str, str]:
|
||||
def _obtain_api_key() -> tuple[str, str, str]:
|
||||
home_resp = get(base_url)
|
||||
if not home_resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain api key")
|
||||
@@ -58,9 +58,10 @@ def _obtain_api_key() -> tuple[str, str]:
|
||||
)
|
||||
if not token_resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain api key")
|
||||
session_id = token_resp.cookies["sessionid"]
|
||||
scraped_api_key = token_resp.json()["token"]["token"]
|
||||
|
||||
return csrf_token, scraped_api_key
|
||||
return csrf_token, session_id, scraped_api_key
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams"):
|
||||
@@ -72,7 +73,7 @@ def request(query: str, params: "OnlineParams"):
|
||||
|
||||
params["headers"].update(headers)
|
||||
else:
|
||||
csrf_token, scraped_api_key = _obtain_api_key()
|
||||
csrf_token, session_id, scraped_api_key = _obtain_api_key()
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {scraped_api_key}",
|
||||
@@ -80,11 +81,10 @@ def request(query: str, params: "OnlineParams"):
|
||||
}
|
||||
|
||||
params["headers"].update(headers)
|
||||
params["cookies"] = {"csrftoken": csrf_token}
|
||||
params["cookies"] = {"csrftoken": session_id, "sessionid": session_id}
|
||||
|
||||
params["url"] = f"{base_url}/api/v1/_search"
|
||||
params["method"] = "POST"
|
||||
params["impersonate"] = "none"
|
||||
|
||||
json_data = {
|
||||
"query": query,
|
||||
|
||||
@@ -78,7 +78,7 @@ time_range_dict = {'day': '24h', 'week': '1w', 'month': '1m', 'year': '1y'}
|
||||
base_url = "https://www.chinaso.com"
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if chinaso_news_source not in t.get_args(ChinasoNewsSourceType):
|
||||
raise ValueError(f"Unsupported news source: {chinaso_news_source}")
|
||||
|
||||
|
||||
@@ -74,7 +74,6 @@ Implementations
|
||||
===============
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
import re
|
||||
from os.path import expanduser, isabs, realpath, commonprefix
|
||||
from shlex import split as shlex_split
|
||||
@@ -84,7 +83,6 @@ from threading import Thread
|
||||
from searx import logger
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
|
||||
engine_type = 'offline'
|
||||
paging = True
|
||||
command = []
|
||||
@@ -101,7 +99,7 @@ _command_logger = logger.getChild('command')
|
||||
_compiled_parse_regex = {}
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
|
||||
def init(engine_settings):
|
||||
check_parsing_options(engine_settings)
|
||||
|
||||
if 'command' not in engine_settings:
|
||||
|
||||
@@ -141,13 +141,12 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
if name:
|
||||
authors.add(name)
|
||||
|
||||
tag = result.get("fieldOfStudy")
|
||||
res.add(
|
||||
res.types.Paper(
|
||||
title=result.get("title"),
|
||||
url=url,
|
||||
content=result.get("fullText", "") or "",
|
||||
tags=[tag] if tag else [],
|
||||
tags=result.get("fieldOfStudy", []),
|
||||
publishedDate=published_date,
|
||||
type=result.get("documentType", "") or "",
|
||||
authors=authors,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Docker Hub (IT)"""
|
||||
|
||||
# pylint: disable=use-dict-literal
|
||||
|
||||
from urllib.parse import urlencode
|
||||
|
||||
@@ -8,9 +8,6 @@ import typing as t
|
||||
from datetime import datetime, timezone
|
||||
import html
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.network import post
|
||||
from searx.utils import format_duration, html_to_text, humanize_number
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
@@ -38,36 +35,15 @@ dogpile_categ = "search"
|
||||
base_url = "https://www.dogpile.com"
|
||||
safe_search_map = {0: "none", 1: "moderate", 2: "heavy"}
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache for the API token from dogpile"""
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if dogpile_categ not in ("search", "images", "videos", "news"):
|
||||
raise ValueError("invalid search type: %s" % dogpile_categ)
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache("dogpile") # one token for images/videos/news
|
||||
return True
|
||||
|
||||
|
||||
def _obtain_token() -> str:
|
||||
token = CACHE.get("token")
|
||||
if token:
|
||||
return token
|
||||
resp = post(f"{base_url}/api/token/refresh", headers={"Origin": base_url}, cookies={"dp_api_token": "1"})
|
||||
if not resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain dogpile token")
|
||||
token = resp.json()["token"]
|
||||
CACHE.set("token", token, expire=240) # 300s ttl
|
||||
return token
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams"):
|
||||
params["url"] = f"{base_url}/api/{dogpile_categ}"
|
||||
params["headers"]["Origin"] = base_url
|
||||
params["cookies"]["dp_api_token"] = "1"
|
||||
if dogpile_categ != "search": # web doesnt need token
|
||||
params["headers"]["x-dogpile-token"] = _obtain_token()
|
||||
|
||||
params["method"] = "POST"
|
||||
params["json"] = {"q": query, "qadf": safe_search_map[params["safesearch"]], "page": params["pageno"]}
|
||||
|
||||
@@ -164,6 +164,7 @@ Terms / phrases that you keep coming across:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Accept-Language
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=global-statement
|
||||
|
||||
import json
|
||||
|
||||
@@ -12,6 +12,7 @@ least we could not find out how language support should work. It seems that
|
||||
most of the features are based on English terms.
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from urllib.parse import urlencode, urlparse, urljoin
|
||||
|
||||
@@ -10,8 +10,7 @@ from datetime import datetime
|
||||
from urllib.parse import urlencode
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from searx.result_types import EngineResults, MainResult, LegacyResult, Image
|
||||
from searx.utils import html_to_text, gen_useragent, extr
|
||||
from searx.utils import get_embeded_stream_url, html_to_text, gen_useragent, extr
|
||||
from searx.network import get # see https://github.com/searxng/searxng/issues/762
|
||||
|
||||
from searx.engines.duckduckgo import fetch_traits # pylint: disable=unused-import
|
||||
@@ -48,7 +47,7 @@ _HTTP_User_Agent: str = gen_useragent()
|
||||
send_accept_language_header = False
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
|
||||
def init(engine_settings: dict[str, t.Any]):
|
||||
|
||||
if engine_settings["ddg_category"] not in ["images", "videos", "news"]:
|
||||
raise ValueError(f"Unsupported DuckDuckGo category: {engine_settings['ddg_category']}")
|
||||
@@ -98,7 +97,6 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
# The vqd value is generated from the query and the UA header. To be able to
|
||||
# reuse the vqd value, the UA header must be static.
|
||||
headers["User-Agent"] = _HTTP_User_Agent
|
||||
params["impersonate"] = "none"
|
||||
vqd = get_vqd(query=query, params=params) or fetch_vqd(query=query, params=params)
|
||||
|
||||
headers["Accept"] = "*/*"
|
||||
@@ -150,51 +148,54 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
|
||||
def _image_result(result):
|
||||
return Image(
|
||||
url=result['url'],
|
||||
title=result['title'],
|
||||
content='',
|
||||
thumbnail_src=result['thumbnail'],
|
||||
img_src=result['image'],
|
||||
resolution='%s x %s' % (result['width'], result['height']),
|
||||
source=result['source'],
|
||||
)
|
||||
return {
|
||||
'template': 'images.html',
|
||||
'url': result['url'],
|
||||
'title': result['title'],
|
||||
'content': '',
|
||||
'thumbnail_src': result['thumbnail'],
|
||||
'img_src': result['image'],
|
||||
'resolution': '%s x %s' % (result['width'], result['height']),
|
||||
'source': result['source'],
|
||||
}
|
||||
|
||||
|
||||
def _video_result(result):
|
||||
return LegacyResult(
|
||||
template='videos.html',
|
||||
url=result['content'],
|
||||
title=result['title'],
|
||||
content=result['description'],
|
||||
thumbnail=result['images'].get('small') or result['images'].get('medium'),
|
||||
source=result['provider'],
|
||||
length=result['duration'],
|
||||
metadata=result.get('uploader'),
|
||||
)
|
||||
return {
|
||||
'template': 'videos.html',
|
||||
'url': result['content'],
|
||||
'title': result['title'],
|
||||
'content': result['description'],
|
||||
'thumbnail': result['images'].get('small') or result['images'].get('medium'),
|
||||
'iframe_src': get_embeded_stream_url(result['content']),
|
||||
'source': result['provider'],
|
||||
'length': result['duration'],
|
||||
'metadata': result.get('uploader'),
|
||||
}
|
||||
|
||||
|
||||
def _news_result(result):
|
||||
return MainResult(
|
||||
url=result['url'],
|
||||
title=result['title'],
|
||||
content=html_to_text(result['excerpt']),
|
||||
publishedDate=datetime.fromtimestamp(result['date']),
|
||||
)
|
||||
return {
|
||||
'url': result['url'],
|
||||
'title': result['title'],
|
||||
'content': html_to_text(result['excerpt']),
|
||||
'source': result['source'],
|
||||
'publishedDate': datetime.fromtimestamp(result['date']),
|
||||
}
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res = EngineResults()
|
||||
def response(resp):
|
||||
results = []
|
||||
res_json = resp.json()
|
||||
|
||||
for result in res_json['results']:
|
||||
if ddg_category == 'images':
|
||||
res.add(_image_result(result))
|
||||
results.append(_image_result(result))
|
||||
elif ddg_category == 'videos':
|
||||
res.add(_video_result(result))
|
||||
results.append(_video_result(result))
|
||||
elif ddg_category == 'news':
|
||||
res.add(_news_result(result))
|
||||
results.append(_news_result(result))
|
||||
else:
|
||||
raise ValueError(f"Invalid duckduckgo category: {ddg_category}")
|
||||
|
||||
return res
|
||||
return results
|
||||
|
||||
@@ -17,7 +17,6 @@ from searx.result_types import EngineResults
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx import weather
|
||||
|
||||
|
||||
about = {
|
||||
"website": 'https://duckduckgo.com/',
|
||||
"wikidata_id": 'Q12805',
|
||||
@@ -109,19 +108,7 @@ def response(resp: SXNG_Response):
|
||||
|
||||
json_data = loads(resp.text[resp.text.find('\n') + 1 : resp.text.rfind('\n') - 2])
|
||||
|
||||
location = json_data.get("location")
|
||||
if not location:
|
||||
return res
|
||||
|
||||
metadata = json_data.get("weatherAlerts", {}).get("metadata", {})
|
||||
geoloc = weather.GeoLocation(
|
||||
name=location,
|
||||
latitude=metadata.get("latitude"),
|
||||
longitude=metadata.get("longitude"),
|
||||
elevation=0,
|
||||
country_code=metadata.get("language").split("-")[-1],
|
||||
timezone=json_data.get("location"),
|
||||
)
|
||||
geoloc = weather.GeoLocation.by_query(resp.search_params["query"])
|
||||
|
||||
weather_answer = EngineResults.types.WeatherAnswer(
|
||||
current=_weather_data(geoloc, json_data["currentWeather"]),
|
||||
|
||||
@@ -14,12 +14,11 @@ can't build it ourselves and must scrape it from the HTML pages.
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
import re
|
||||
|
||||
from urllib.parse import quote_plus, urljoin
|
||||
from urllib.parse import quote_plus
|
||||
from lxml import html
|
||||
|
||||
from searx.utils import html_to_text, extract_text, eval_xpath
|
||||
from searx.utils import html_to_text, gen_useragent, extract_text, eval_xpath
|
||||
from searx.result_types import EngineResults
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.network import get
|
||||
@@ -39,6 +38,7 @@ about = {
|
||||
# engine dependent config
|
||||
categories = ["general"]
|
||||
paging = True
|
||||
_HTTP_User_Agent: str = gen_useragent()
|
||||
|
||||
base_url = "https://duckduckgo.com"
|
||||
|
||||
@@ -73,8 +73,6 @@ def _fetch_first_page_link(
|
||||
resp = get(
|
||||
url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
|
||||
headers=headers,
|
||||
impersonate="firefox",
|
||||
default_headers=False,
|
||||
timeout=2,
|
||||
)
|
||||
|
||||
@@ -98,43 +96,6 @@ def _cache_key(query: str, pageno: int) -> str:
|
||||
return f"nextpage_url|{query}|{pageno}"
|
||||
|
||||
|
||||
def _solve_jsa(resp: "SXNG_Response") -> "SXNG_Response":
|
||||
"""Duckduckgo sometimes issues a challenge instead of json."""
|
||||
|
||||
# length that a real browser would report for where the broken snippet is
|
||||
html_len = {
|
||||
"<p><div></p><p></div": 32,
|
||||
"<li><div></li><li></div": 29,
|
||||
"<div><div></div><div></div": 33,
|
||||
"<br><div></br><br></div": 23,
|
||||
}
|
||||
|
||||
js = resp.text or ""
|
||||
jsa_match = re.search(r"let jsa = (\d+);.*?DDG\.deep\.initialize\('([^']+)'", js, re.S)
|
||||
if not jsa_match:
|
||||
return resp
|
||||
|
||||
js_functions = dict(re.findall(r"let (\w+) = function\(num\) \{([^}]*)\};", js))
|
||||
jsa = int(jsa_match.group(1))
|
||||
try:
|
||||
for name in re.findall(r"jsa = (\w+)\(jsa\);", js):
|
||||
body = js_functions[name]
|
||||
mul = re.search(r"num \* (\d+)", body)
|
||||
jsa = jsa * int(mul.group(1)) if mul else jsa + html_len[re.search(r"`([^`]+)`", body).group(1)]
|
||||
except (KeyError, AttributeError):
|
||||
return resp
|
||||
|
||||
params = resp.search_params
|
||||
follow = get(
|
||||
urljoin("https://links.duckduckgo.com", jsa_match.group(2) + str(jsa)),
|
||||
headers=params["headers"],
|
||||
impersonate="firefox",
|
||||
default_headers=False,
|
||||
)
|
||||
follow.search_params = params
|
||||
return follow
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
if len(query) >= 500:
|
||||
@@ -142,15 +103,25 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
params["url"] = None
|
||||
return
|
||||
|
||||
# firefox TLS only
|
||||
params["impersonate"] = "firefox"
|
||||
params["default_headers"] = False
|
||||
headers = params["headers"]
|
||||
|
||||
# The vqd value is generated from the query and the UA header. To be able
|
||||
# to reuse the vqd value, the UA header must be static.
|
||||
headers["User-Agent"] = _HTTP_User_Agent
|
||||
headers["Accept"] = "*/*"
|
||||
headers["Referer"] = f"{base_url}/"
|
||||
headers["Host"] = "duckduckgo.com"
|
||||
|
||||
# Sec-Fetch headers are required to not get blocked when sending a Firefox user agent
|
||||
headers["Sec-Fetch-Dest"] = "script"
|
||||
headers["Sec-Fetch-Mode"] = "no-cors"
|
||||
headers["Sec-Fetch-Site"] = "same-site"
|
||||
|
||||
api_url = ""
|
||||
if params["pageno"] > 1:
|
||||
api_url = CACHE.get(_cache_key(query, params["pageno"]))
|
||||
else:
|
||||
api_url = _fetch_first_page_link(query, params["headers"])
|
||||
api_url = _fetch_first_page_link(query, headers)
|
||||
|
||||
if not api_url:
|
||||
params["url"] = None
|
||||
@@ -158,27 +129,14 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
|
||||
|
||||
# loads as a script
|
||||
headers = params["headers"]
|
||||
headers["Accept"] = "*/*"
|
||||
headers["Sec-Fetch-Dest"] = "script"
|
||||
headers["Sec-Fetch-Mode"] = "no-cors"
|
||||
headers["Sec-Fetch-Site"] = "same-site"
|
||||
headers["Referer"] = f"{base_url}/"
|
||||
|
||||
# TODO: support safesearch, timerange and engine traits # pylint:disable=fixme
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
res = EngineResults()
|
||||
res_json = resp.json()
|
||||
|
||||
# check if ddg returns a challenge
|
||||
# e.g. 'site:github.com searxng'
|
||||
if "let jsa =" in (resp.text or ""):
|
||||
resp = _solve_jsa(resp)
|
||||
|
||||
results = resp.json()["results"]
|
||||
for result in results:
|
||||
for result in res_json["results"]:
|
||||
if "u" not in result:
|
||||
continue
|
||||
|
||||
@@ -186,13 +144,13 @@ def response(resp: "SXNG_Response"):
|
||||
res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
|
||||
)
|
||||
|
||||
if results:
|
||||
next_page_path = results[-1].get("n")
|
||||
if next_page_path:
|
||||
CACHE.set(
|
||||
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
|
||||
base_url + next_page_path,
|
||||
expire=60 * 60,
|
||||
)
|
||||
# link to next page
|
||||
next_page_path = res_json["results"][-1].get("n")
|
||||
if next_page_path:
|
||||
CACHE.set(
|
||||
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
|
||||
base_url + next_page_path,
|
||||
expire=60 * 60,
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# pylint: disable=invalid-name
|
||||
"""Dummy Offline"""
|
||||
|
||||
|
||||
# about
|
||||
about = {
|
||||
"wikidata_id": None,
|
||||
|
||||
@@ -41,7 +41,6 @@ authentication configured to read from ``my-index`` index.
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
from json import loads, dumps
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.result_types import EngineResults
|
||||
@@ -69,7 +68,7 @@ show_metadata = False
|
||||
page_size = 10
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
|
||||
def init(engine_settings):
|
||||
if 'query_type' in engine_settings and engine_settings['query_type'] not in _available_query_types:
|
||||
raise ValueError('unsupported query type', engine_settings['query_type'])
|
||||
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""`Europe PMC`_ provides comprehensive access to life sciences literature from
|
||||
trusted sources. With Europe PMC you can search and read millions of
|
||||
publications, preprints and other documents enriched with links to supporting
|
||||
data, reviews, protocols, and other relevant resources.
|
||||
|
||||
.. _Europe PMC: https://europepmc.org/
|
||||
|
||||
Configuration
|
||||
=============
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: europepmc
|
||||
engine: europepmc
|
||||
shortcut: epmc
|
||||
|
||||
Implementations
|
||||
===============
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from dateutil.parser import isoparse
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import html_to_text
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
|
||||
about = {
|
||||
"website": "https://europepmc.org/",
|
||||
"wikidata_id": "Q5412157",
|
||||
"official_api_documentation": "https://europepmc.org/RestfulWebService",
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
categories = ["science", "scientific publications"]
|
||||
paging = True
|
||||
|
||||
# engine dependent config
|
||||
search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
|
||||
article_url = "https://europepmc.org/article/"
|
||||
|
||||
page_size = 20
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache for storing the pagination cursor."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]):
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
|
||||
|
||||
def _cache_key(query: str, page: int) -> str:
|
||||
return f"{query}|{page}"
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
args = {
|
||||
"query": query,
|
||||
"format": "json",
|
||||
"resultType": "core",
|
||||
"pageSize": page_size,
|
||||
}
|
||||
|
||||
if params["pageno"] > 1:
|
||||
if cursor := CACHE.get(_cache_key(query, params["pageno"])):
|
||||
args["cursorMark"] = cursor
|
||||
else:
|
||||
# no cached cursor for that page
|
||||
params["url"] = None
|
||||
return
|
||||
|
||||
params["url"] = f"{search_url}?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
json_resp = resp.json()
|
||||
|
||||
# store pagination cursor for loading next pages in cache
|
||||
if next_cursor := json_resp.get("nextCursorMark"):
|
||||
next_page = resp.search_params["pageno"] + 1
|
||||
query = resp.search_params["query"]
|
||||
CACHE.set(_cache_key(query, next_page), next_cursor)
|
||||
|
||||
all_results = json_resp.get("resultList", {}).get("result", [])
|
||||
|
||||
for item in all_results:
|
||||
source = item.get("source", "")
|
||||
identifier = item.get("id", "")
|
||||
url = f"{article_url}{source}/{identifier}" if source and identifier else ""
|
||||
|
||||
journal_info: dict[str, t.Any] = item.get("journalInfo", {})
|
||||
journal: dict[str, t.Any] = journal_info.get("journal", {})
|
||||
|
||||
res.add(
|
||||
res.types.Paper(
|
||||
url=url,
|
||||
title=html_to_text(item.get("title", "")),
|
||||
content=html_to_text(item.get("abstractText", "")),
|
||||
journal=journal.get("title", ""),
|
||||
issn=[journal.get("issn", "")],
|
||||
authors=_get_authors(item),
|
||||
doi=item.get("doi", ""),
|
||||
publishedDate=_get_published_date(item),
|
||||
type=", ".join((item.get("pubTypeList", {})).get("pubType", [])),
|
||||
pdf_url=_get_pdf_url(item),
|
||||
html_url=url,
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def _get_authors(item: dict[str, t.Any]) -> list:
|
||||
"""Extract the list of authors from the item."""
|
||||
if authors := item.get("authorString", None):
|
||||
authors = [author.strip().rstrip(".") for author in authors.split(",") if author.strip()]
|
||||
else:
|
||||
authors = []
|
||||
return authors
|
||||
|
||||
|
||||
def _get_pdf_url(item: dict[str, t.Any]) -> str:
|
||||
"""Extract the PDF URL in case it is open access."""
|
||||
for url_info in (item.get("fullTextUrlList", {})).get("fullTextUrl", []):
|
||||
if url_info.get("documentStyle") == "pdf" and url_info.get("availabilityCode") == "OA":
|
||||
return url_info.get("url", "")
|
||||
return ""
|
||||
|
||||
|
||||
def _get_published_date(item: dict[str, t.Any]) -> datetime | None:
|
||||
"""Extract the published date from the item and convert it to a datetime object."""
|
||||
if unformatted_date := item.get("firstPublicationDate"):
|
||||
return isoparse(unformatted_date)
|
||||
return None
|
||||
@@ -78,7 +78,7 @@ content_max_characters: int = 500
|
||||
"""Maximum characters for the requested content."""
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if not api_key:
|
||||
raise SearxEngineAPIException("No API key provided")
|
||||
if not 1 <= results_per_page <= 100:
|
||||
|
||||
@@ -47,7 +47,7 @@ CACHE_VALID_DURATION = 30 * 24 * 3600 # one month, same as website
|
||||
"""Duration how long settings cookies are valid."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
|
||||
def init(engine_settings: dict[str, t.Any]):
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ Implementation
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
from urllib.parse import urlencode
|
||||
from dateutil import parser
|
||||
|
||||
@@ -78,7 +77,7 @@ page_size: int = 10
|
||||
"""Maximum number of results per page (default 10)."""
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if not base_url:
|
||||
raise ValueError('gitea engine: base_url is unset')
|
||||
|
||||
|
||||
@@ -65,7 +65,6 @@ code lines are just relabeled (starting from 1) and appended (a disjoint set of
|
||||
code blocks in a single file might be returned from the API).
|
||||
"""
|
||||
|
||||
|
||||
import typing as t
|
||||
from urllib.parse import urlencode
|
||||
|
||||
|
||||
@@ -9,15 +9,12 @@ engines:
|
||||
- :ref:`google scholar engine`
|
||||
- :ref:`google autocomplete`
|
||||
|
||||
This implementation uses Nokia user agents to request an XML layout from Google.
|
||||
The normal web version requires executing JavaScript to load the results and
|
||||
therefore is currently not used here. See `Google discussion`_ for more
|
||||
information on that topic.
|
||||
|
||||
.. _Google discussion: https://github.com/searxng/searxng/issues/6359
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import time
|
||||
import typing as t
|
||||
from urllib.parse import unquote, urlencode
|
||||
|
||||
@@ -47,16 +44,16 @@ about = {
|
||||
"official_api_documentation": "https://developers.google.com/custom-search/",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "XML",
|
||||
"results": "HTML",
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ["general", "web"]
|
||||
paging = True
|
||||
max_page = 50
|
||||
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
|
||||
"""`Google max 50 pages`_
|
||||
|
||||
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
|
||||
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
@@ -67,23 +64,38 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
|
||||
# Filter results. 0: None, 1: Moderate, 2: Strict
|
||||
filter_mapping = {0: "off", 1: "medium", 2: "high"}
|
||||
|
||||
# https://github.com/searxng/searxng/issues/6359
|
||||
nokia_useragents = (
|
||||
"Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
|
||||
"Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
|
||||
"Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
|
||||
"Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
|
||||
"Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
|
||||
"NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
|
||||
)
|
||||
|
||||
|
||||
# specific xpath variables
|
||||
# ------------------------
|
||||
|
||||
# Suggestions are links placed in a *card-section*, we extract only the text
|
||||
# from the links not the links itself.
|
||||
suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
|
||||
suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
|
||||
|
||||
|
||||
_arcid_range = string.ascii_letters + string.digits + "_-"
|
||||
_arcid_random: tuple[str, int] | None = None
|
||||
|
||||
|
||||
def ui_async(start: int) -> str:
|
||||
"""Format of the response from UI's async request.
|
||||
|
||||
- ``arc_id:<...>,use_ac:true,_fmt:prog``
|
||||
|
||||
The arc_id is random generated every hour.
|
||||
"""
|
||||
global _arcid_random # pylint: disable=global-statement
|
||||
|
||||
use_ac = "use_ac:true"
|
||||
# _fmt:html returns a HTTP 500 when user search for celebrities like
|
||||
# '!google natasha allegri' or '!google chris evans'
|
||||
_fmt = "_fmt:prog"
|
||||
|
||||
# create a new random arc_id every hour
|
||||
if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
|
||||
_arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
|
||||
arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
|
||||
|
||||
return ",".join([arc_id, use_ac, _fmt])
|
||||
|
||||
|
||||
def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
|
||||
@@ -115,11 +127,19 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
A instance of :py:obj:`babel.core.Locale` build from the
|
||||
``searxng_locale`` value.
|
||||
|
||||
subdomain:
|
||||
Google subdomain :py:obj:`google_domains` that fits to the country
|
||||
code.
|
||||
|
||||
params:
|
||||
Py-Dictionary with additional request arguments (can be passed to
|
||||
:py:func:`urllib.parse.urlencode`).
|
||||
|
||||
- ``hl`` parameter: specifies the interface language of user interface.
|
||||
- ``lr`` parameter: restricts search results to documents written in
|
||||
a particular language.
|
||||
- ``cr`` parameter: restricts search results to documents
|
||||
originating in a particular country.
|
||||
- ``ie`` parameter: sets the character encoding scheme that should
|
||||
be used to interpret the query string ('utf8').
|
||||
- ``oe`` parameter: sets the character encoding scheme that should
|
||||
@@ -136,6 +156,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
ret_val: dict[str, t.Any] = {
|
||||
"language": None,
|
||||
"country": None,
|
||||
"subdomain": None,
|
||||
"params": {},
|
||||
"headers": {},
|
||||
"cookies": {},
|
||||
@@ -148,7 +169,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
except babel.core.UnknownLocaleError:
|
||||
locale = None
|
||||
|
||||
eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
|
||||
eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
|
||||
lang_code = eng_lang.split("_")[-1] # lang_zh-TW --> zh-TW / lang_en --> en
|
||||
country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
|
||||
|
||||
@@ -163,6 +184,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
ret_val["language"] = eng_lang
|
||||
ret_val["country"] = country
|
||||
ret_val["locale"] = locale
|
||||
ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
|
||||
|
||||
# hl parameter:
|
||||
# The hl parameter specifies the interface language (host language) of
|
||||
@@ -201,11 +223,9 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
|
||||
# specify a region (country) only if a region is given in the selected
|
||||
# locale --> https://github.com/searxng/searxng/issues/2672
|
||||
|
||||
if country is not None:
|
||||
ret_val["params"]["cr"] = ""
|
||||
if len(sxng_locale.split("-")) > 1:
|
||||
ret_val["params"]["cr"] = "country" + country
|
||||
ret_val["params"]["cr"] = ""
|
||||
if len(sxng_locale.split("-")) > 1:
|
||||
ret_val["params"]["cr"] = "country" + country
|
||||
|
||||
# gl parameter: (mandatory by Google News)
|
||||
# The gl parameter value is a two-letter country code. For WebSearch
|
||||
@@ -280,78 +300,88 @@ def detect_google_sorry(resp: "SXNG_Response"):
|
||||
raise SearxEngineCaptchaException()
|
||||
|
||||
|
||||
def unwrap_google_url(raw_url: str) -> str:
|
||||
# remove redirector from url
|
||||
if raw_url.startswith("/url?q="):
|
||||
return unquote(raw_url[7:].split("&sa=U")[0])
|
||||
return raw_url
|
||||
|
||||
|
||||
def wml_dom(resp: "SXNG_Response"):
|
||||
detect_google_sorry(resp)
|
||||
text = resp.text
|
||||
if text.lstrip().startswith("<?xml"):
|
||||
text = text.split("?>", 1)[-1]
|
||||
return html.fromstring(text)
|
||||
|
||||
|
||||
def google_request(
|
||||
query: str,
|
||||
params: "OnlineParams",
|
||||
extra_args: dict[str, t.Any] | None = None,
|
||||
*,
|
||||
eng_traits: EngineTraits | None = None,
|
||||
use_time_range: bool = True,
|
||||
use_safesearch: bool = True,
|
||||
safesearch_map: dict[int, str] | None = None,
|
||||
use_locales: bool = True,
|
||||
) -> None:
|
||||
google_info = get_google_info(params, eng_traits or traits)
|
||||
if not use_locales:
|
||||
google_info["params"].pop("lr")
|
||||
google_info["params"].pop("cr")
|
||||
|
||||
start = (params["pageno"] - 1) * 10
|
||||
args: dict[str, t.Any] = {
|
||||
"q": query,
|
||||
"sca_esv": "1",
|
||||
**google_info["params"],
|
||||
**(extra_args or {}),
|
||||
}
|
||||
if start:
|
||||
args["start"] = start
|
||||
if use_time_range and params["time_range"] in time_range_dict:
|
||||
args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
|
||||
if use_safesearch and params["safesearch"]:
|
||||
args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
|
||||
|
||||
params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
|
||||
params["headers"]["User-Agent"] = random.choice(nokia_useragents)
|
||||
params["impersonate"] = "chrome99_android"
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
google_request(query, params)
|
||||
"""Google search request"""
|
||||
# pylint: disable=line-too-long
|
||||
start = (params["pageno"] - 1) * 10
|
||||
google_info = get_google_info(params, traits)
|
||||
|
||||
# https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
|
||||
query_url = (
|
||||
"https://"
|
||||
+ google_info["subdomain"]
|
||||
+ "/search"
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
"q": query,
|
||||
**google_info["params"],
|
||||
"filter": "0",
|
||||
"start": start,
|
||||
# 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
|
||||
# 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
|
||||
# 'cs' : 1,
|
||||
# 'sa': 'N',
|
||||
# 'yv': 3,
|
||||
# 'prmd': 'vin',
|
||||
# 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
|
||||
# 'sa': 'N',
|
||||
# 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
|
||||
# formally known as use_mobile_ui
|
||||
# "asearch": "arc",
|
||||
# "async": str_async,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if params["time_range"] in time_range_dict:
|
||||
query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
|
||||
if params["safesearch"]:
|
||||
query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
|
||||
params["url"] = query_url
|
||||
|
||||
params["cookies"] = google_info["cookies"]
|
||||
params["headers"].update(google_info["headers"])
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
# regex match to get image map that is found inside the returned javascript:
|
||||
# (function(){var s='...';var i=['...'] ...}
|
||||
RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
|
||||
|
||||
|
||||
def parse_url_images(text: str):
|
||||
data_image_map = {}
|
||||
|
||||
for image_url, img_id in RE_DATA_IMAGE.findall(text):
|
||||
data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
|
||||
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
|
||||
return data_image_map
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
"""Get response from google's search request"""
|
||||
# pylint: disable=too-many-branches, too-many-statements
|
||||
detect_google_sorry(resp)
|
||||
data_image_map = parse_url_images(resp.text)
|
||||
|
||||
results = EngineResults()
|
||||
dom = wml_dom(resp)
|
||||
|
||||
# convert the text to dom
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
# parse results
|
||||
for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
|
||||
for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
|
||||
# pylint: disable=too-many-nested-blocks
|
||||
|
||||
try:
|
||||
title_tag = eval_xpath_getindex(
|
||||
result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
|
||||
)
|
||||
title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
|
||||
if title_tag is None:
|
||||
# this not one of the common google results *section*
|
||||
logger.debug("ignoring item from the result_xpath list: missing title")
|
||||
continue
|
||||
title = extract_text(title_tag)
|
||||
|
||||
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
|
||||
raw_url = result.get("href")
|
||||
if raw_url is None:
|
||||
logger.debug(
|
||||
'ignoring item from the result_xpath list: missing url of title "%s"',
|
||||
@@ -359,19 +389,30 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
)
|
||||
continue
|
||||
|
||||
url = unwrap_google_url(raw_url)
|
||||
content = extract_text(
|
||||
eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
|
||||
)
|
||||
thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
url=url,
|
||||
title=title or "",
|
||||
content=content or "",
|
||||
thumbnail=thumbnail or "",
|
||||
)
|
||||
)
|
||||
if raw_url.startswith('/url?q='):
|
||||
url = unquote(raw_url[7:].split("&sa=U")[0]) # remove the google redirector
|
||||
else:
|
||||
url = raw_url
|
||||
|
||||
content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
|
||||
for item in content_nodes:
|
||||
for script in item.xpath(".//script"):
|
||||
script.getparent().remove(script)
|
||||
|
||||
content = extract_text(content_nodes[0])
|
||||
|
||||
# Images that are NOT the favicon
|
||||
xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
|
||||
|
||||
thumbnail = None
|
||||
if xpath_image is not None:
|
||||
thumbnail = xpath_image.get("src")
|
||||
if thumbnail.startswith("data:image"):
|
||||
img_id = xpath_image.get("id")
|
||||
if img_id:
|
||||
thumbnail = data_image_map.get(img_id)
|
||||
|
||||
results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
|
||||
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
logger.error(e, exc_info=True)
|
||||
@@ -379,8 +420,10 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
|
||||
# parse suggestion
|
||||
for suggestion in eval_xpath_list(dom, suggestion_xpath):
|
||||
results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
|
||||
# append suggestion
|
||||
results.append({"suggestion": extract_text(suggestion)})
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
||||
|
||||
@@ -413,12 +456,14 @@ skip_countries = [
|
||||
]
|
||||
|
||||
|
||||
def fetch_traits(engine_traits: EngineTraits):
|
||||
def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
|
||||
"""Fetch languages from Google."""
|
||||
# pylint: disable=import-outside-toplevel, too-many-branches
|
||||
|
||||
from searx.network import get # see https://github.com/searxng/searxng/issues/762
|
||||
|
||||
engine_traits.custom["supported_domains"] = {}
|
||||
|
||||
resp = get("https://www.google.com/preferences", timeout=5)
|
||||
if not resp.ok:
|
||||
raise RuntimeError("Response from Google preferences is not OK.")
|
||||
@@ -469,3 +514,22 @@ def fetch_traits(engine_traits: EngineTraits):
|
||||
|
||||
# alias regions
|
||||
engine_traits.regions["zh-CN"] = "HK"
|
||||
|
||||
# supported domains
|
||||
|
||||
if add_domains:
|
||||
resp = get("https://www.google.com/supported_domains", timeout=5)
|
||||
if not resp.ok:
|
||||
raise RuntimeError("Response from Google supported domains is not OK.")
|
||||
|
||||
for domain in resp.text.split():
|
||||
domain = domain.strip()
|
||||
if not domain or domain in [
|
||||
".google.com",
|
||||
]:
|
||||
continue
|
||||
region = domain.split(".")[-1].upper()
|
||||
engine_traits.custom["supported_domains"][region] = "www" + domain
|
||||
if region == "HK":
|
||||
# There is no google.cn, we use .com.hk for zh-CN
|
||||
engine_traits.custom["supported_domains"]["CN"] = "www" + domain
|
||||
|
||||
@@ -30,7 +30,6 @@ about = {
|
||||
|
||||
categories = ["general", "web"]
|
||||
paging = True
|
||||
enable_http3 = True
|
||||
max_page = 5
|
||||
page_size = 20
|
||||
time_range_support = True
|
||||
@@ -96,11 +95,12 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
token = _cse_token()
|
||||
|
||||
google_info = get_google_info(params, traits)
|
||||
info: dict[str, str] = google_info["params"]
|
||||
|
||||
args = {
|
||||
"rsz": "filtered_cse",
|
||||
"num": str(page_size),
|
||||
"hl": google_info["params"]["hl"],
|
||||
"hl": info["hl"],
|
||||
"cselibv": token["cselibv"],
|
||||
"cx": CX,
|
||||
"q": query,
|
||||
@@ -114,6 +114,10 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
|
||||
args["sort"] = f"date:r:{start_date}:{end_date}"
|
||||
|
||||
if info.get("lr"):
|
||||
args["lr"] = info["lr"]
|
||||
if info.get("cr"):
|
||||
args["cr"] = info["cr"]
|
||||
if google_info["country"] not in (None, "ZZ"):
|
||||
args["gl"] = google_info["country"]
|
||||
if token["exp"]:
|
||||
|
||||
@@ -1,76 +1,122 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Google Images: see :py:obj:`searx.engines.google`."""
|
||||
"""This is the implementation of the Google Images engine using the internal
|
||||
Google API used by the Google Go Android app.
|
||||
|
||||
import typing as t
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
This internal API offer results in
|
||||
|
||||
- JSON (``_fmt:json``)
|
||||
- Protobuf_ (``_fmt:pb``)
|
||||
- Protobuf_ compressed? (``_fmt:pc``)
|
||||
- HTML (``_fmt:html``)
|
||||
- Protobuf_ encoded in JSON (``_fmt:jspb``).
|
||||
|
||||
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode
|
||||
from json import loads
|
||||
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import google_request, wml_dom
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import eval_xpath_list
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
from searx.engines.google import (
|
||||
get_google_info,
|
||||
time_range_dict,
|
||||
detect_google_sorry,
|
||||
)
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": "https://images.google.com",
|
||||
"wikidata_id": "Q521550",
|
||||
"official_api_documentation": "https://developers.google.com/custom-search",
|
||||
"website": 'https://images.google.com',
|
||||
"wikidata_id": 'Q521550',
|
||||
"official_api_documentation": 'https://developers.google.com/custom-search',
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "XML",
|
||||
"results": 'JSON',
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ["images", "web"]
|
||||
categories = ['images', 'web']
|
||||
paging = True
|
||||
enable_http3 = True
|
||||
max_page = 50
|
||||
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
|
||||
"""`Google max 50 pages`_
|
||||
|
||||
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
|
||||
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
safesearch = True
|
||||
|
||||
filter_mapping = {0: "images", 1: "active", 2: "active"}
|
||||
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
google_request(
|
||||
query,
|
||||
params,
|
||||
{"tbm": "isch"},
|
||||
eng_traits=traits,
|
||||
safesearch_map=filter_mapping,
|
||||
use_locales=False,
|
||||
def request(query, params):
|
||||
"""Google-Image search request"""
|
||||
|
||||
google_info = get_google_info(params, traits)
|
||||
|
||||
query_url = (
|
||||
'https://'
|
||||
+ google_info['subdomain']
|
||||
+ '/search'
|
||||
+ '?'
|
||||
+ urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
|
||||
# don't urlencode this because wildly different AND bad results
|
||||
# pagination uses Zero-based numbering
|
||||
+ f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
|
||||
)
|
||||
|
||||
if params['time_range'] in time_range_dict:
|
||||
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
|
||||
if params['safesearch']:
|
||||
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
|
||||
params['url'] = query_url
|
||||
params['cookies'] = google_info['cookies']
|
||||
params['headers'].update(google_info['headers'])
|
||||
# this ua will allow getting ~50 results instead of 10. #1641
|
||||
params['headers']['User-Agent'] = (
|
||||
'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
|
||||
)
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
results = EngineResults()
|
||||
dom = wml_dom(resp)
|
||||
return params
|
||||
|
||||
for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
|
||||
qs = parse_qs(urlparse(link.get("href", "")).query)
|
||||
img_src = qs.get("imgurl", [""])[0]
|
||||
url = qs.get("imgrefurl", [""])[0]
|
||||
if not img_src or not url:
|
||||
continue
|
||||
width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
|
||||
tbnid = qs.get("tbnid", [""])[0]
|
||||
results.add(
|
||||
results.types.Image(
|
||||
url=url,
|
||||
title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
|
||||
img_src=img_src,
|
||||
thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
|
||||
resolution=f"{width} x {height}" if width and height else "",
|
||||
)
|
||||
)
|
||||
|
||||
def response(resp):
|
||||
"""Get response from google's search request"""
|
||||
results = []
|
||||
|
||||
detect_google_sorry(resp)
|
||||
|
||||
json_start = resp.text.find('{"ischj":')
|
||||
json_data = loads(resp.text[json_start:])
|
||||
|
||||
for item in json_data["ischj"].get("metadata", []):
|
||||
result_item = {
|
||||
'url': item["result"]["referrer_url"],
|
||||
'title': item["result"]["page_title"],
|
||||
'content': item["text_in_grid"]["snippet"],
|
||||
'source': item["result"]["site_title"],
|
||||
'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
|
||||
'img_src': item["original_image"]["url"],
|
||||
'thumbnail_src': item["thumbnail"]["url"],
|
||||
'template': 'images.html',
|
||||
}
|
||||
|
||||
author = item["result"].get('iptc', {}).get('creator')
|
||||
if author:
|
||||
result_item['author'] = ', '.join(author)
|
||||
|
||||
copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
|
||||
if copyright_notice:
|
||||
result_item['source'] += ' | ' + copyright_notice
|
||||
|
||||
freshness_date = item["result"].get("freshness_date")
|
||||
if freshness_date:
|
||||
result_item['source'] += ' | ' + freshness_date
|
||||
|
||||
file_size = item.get('gsa', {}).get('file_size')
|
||||
if file_size:
|
||||
result_item['source'] += ' (%s)' % file_size
|
||||
|
||||
results.append(result_item)
|
||||
|
||||
return results
|
||||
|
||||
@@ -1,91 +1,325 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Google News: see :py:obj:`searx.engines.google`."""
|
||||
"""This is the implementation of the Google News engine.
|
||||
|
||||
Google News has a different region handling compared to Google WEB.
|
||||
|
||||
- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
|
||||
- the hl_ argument has to be set correctly (and different to Google WEB)
|
||||
- the gl_ argument is mandatory
|
||||
|
||||
If one of this argument is not set correctly, the request is redirected to
|
||||
CONSENT dialog::
|
||||
|
||||
https://consent.google.com/m?continue=
|
||||
|
||||
The google news API ignores some parameters from the common :ref:`google API`:
|
||||
|
||||
- num_ : the number of search results is ignored / there is no paging all
|
||||
results for a query term are in the first response.
|
||||
- save_ : is ignored / Google-News results are always *SafeSearch*
|
||||
|
||||
.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
|
||||
.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
|
||||
.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
|
||||
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import google_request, unwrap_google_url, wml_dom
|
||||
from searx.result_types import EngineResults
|
||||
import json
|
||||
import base64
|
||||
from urllib.parse import urlencode
|
||||
from lxml import html
|
||||
import babel
|
||||
|
||||
from searx import locales
|
||||
from searx.utils import (
|
||||
eval_xpath_getindex,
|
||||
eval_xpath,
|
||||
eval_xpath_list,
|
||||
eval_xpath_getindex,
|
||||
extract_text,
|
||||
)
|
||||
|
||||
from searx.engines.google import fetch_traits as _fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import (
|
||||
get_google_info,
|
||||
detect_google_sorry,
|
||||
)
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": "https://www.google.com",
|
||||
"website": "https://news.google.com",
|
||||
"wikidata_id": "Q12020",
|
||||
"official_api_documentation": "https://developers.google.com/custom-search",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "XML",
|
||||
"results": "HTML",
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ["news"]
|
||||
paging = True
|
||||
max_page = 50
|
||||
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
|
||||
|
||||
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
paging = False
|
||||
time_range_support = False
|
||||
language_support = True
|
||||
safesearch = False
|
||||
|
||||
# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
|
||||
# False here.
|
||||
#
|
||||
# safesearch : results are identical for safesearch=0 and safesearch=2
|
||||
safesearch = True
|
||||
base_url: str = "https://news.google.com"
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
google_request(
|
||||
query,
|
||||
params,
|
||||
{"tbm": "nws"},
|
||||
eng_traits=traits,
|
||||
use_time_range=False,
|
||||
use_safesearch=False,
|
||||
use_locales=False,
|
||||
"""Google-News search request"""
|
||||
|
||||
sxng_locale = params.get("searxng_locale", "en-US")
|
||||
ceid: str = locales.get_engine_locale(
|
||||
sxng_locale, traits.custom["ceid"], default="US:en"
|
||||
) # pyright: ignore[reportAssignmentType]
|
||||
google_info = get_google_info(params, traits)
|
||||
google_info["subdomain"] = "news.google.com" # google news has only one domain
|
||||
|
||||
ceid_region, ceid_lang = ceid.split(":")
|
||||
ceid_lang, ceid_suffix = (
|
||||
ceid_lang.split(":")
|
||||
+ [
|
||||
"",
|
||||
]
|
||||
)[:2]
|
||||
|
||||
google_info["params"]["hl"] = ceid_lang
|
||||
|
||||
if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
|
||||
|
||||
if ceid_region.lower() == ceid_lang:
|
||||
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
|
||||
else:
|
||||
google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
|
||||
|
||||
elif ceid_region.lower() != ceid_lang:
|
||||
|
||||
if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
|
||||
google_info["params"]["hl"] = ceid_lang
|
||||
else:
|
||||
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
|
||||
|
||||
google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
|
||||
google_info["params"]["gl"] = ceid_region
|
||||
|
||||
query_url = (
|
||||
"https://"
|
||||
+ google_info["subdomain"]
|
||||
+ "/search?"
|
||||
+ urlencode(
|
||||
{"q": query, **google_info["params"]},
|
||||
)
|
||||
# ceid includes a ':' character which must not be urlencoded
|
||||
+ ("&ceid=%s" % ceid)
|
||||
)
|
||||
|
||||
|
||||
def _span_text(link, css_class: str):
|
||||
return extract_text(
|
||||
eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
|
||||
allow_none=True,
|
||||
)
|
||||
params["url"] = query_url
|
||||
params["cookies"] = google_info["cookies"]
|
||||
params["headers"].update(google_info["headers"])
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
results = EngineResults()
|
||||
seen = set()
|
||||
for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
|
||||
href = link.get("href")
|
||||
if not href:
|
||||
"""Get response from google's search request"""
|
||||
|
||||
res = EngineResults()
|
||||
|
||||
detect_google_sorry(resp)
|
||||
|
||||
# convert the text to dom
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
|
||||
|
||||
url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
|
||||
if not url:
|
||||
continue
|
||||
if url.startswith("./"):
|
||||
url = base_url + url[1:]
|
||||
|
||||
# The real URL is often encoded in the "jslog" attribute
|
||||
jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
|
||||
|
||||
# Try to extract the real URL from jslog
|
||||
real_url: str | None = None
|
||||
if jslog:
|
||||
# jslog format is usually: "95014; 5:<base64>; track:click,vis". We
|
||||
# want the second part (index 1) after splitting by ";"
|
||||
parts: list[str] = jslog.split(";")
|
||||
if len(parts) > 1:
|
||||
b64_data: str = parts[1].split(":")[-1].strip()
|
||||
# Pad base64 if necessary
|
||||
b64_data += "=" * (-len(b64_data) % 4)
|
||||
decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
|
||||
# The URL is typically the last element in the decoded array
|
||||
if (
|
||||
isinstance(decoded_data, list)
|
||||
and isinstance(decoded_data[-1], str)
|
||||
and decoded_data[-1].startswith("http")
|
||||
):
|
||||
real_url = decoded_data[-1]
|
||||
if real_url:
|
||||
url = real_url
|
||||
else:
|
||||
logger.error(f"no real-url found: {url}")
|
||||
continue
|
||||
|
||||
url = unwrap_google_url(href)
|
||||
if url in seen or "google.com/search" in url:
|
||||
continue
|
||||
title = extract_text(eval_xpath(result, "./h4")) or ""
|
||||
|
||||
title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
|
||||
if not title:
|
||||
continue
|
||||
# The pub_date is mostly a string like 'yesterday', not a real timezone
|
||||
# date or time. Therefore we can't use publishedDate and place the
|
||||
# *pub* sting into the content.
|
||||
|
||||
source = _span_text(link, "dXDvrc")
|
||||
pub_date = _span_text(link, "YVIcad")
|
||||
thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
|
||||
pub_date = extract_text(eval_xpath(result, ".//time"))
|
||||
pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
|
||||
content = " / ".join([x for x in [pub_origin, pub_date] if x])
|
||||
|
||||
seen.add(url)
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
|
||||
if thumbnail and thumbnail.startswith("/"):
|
||||
thumbnail = base_url + thumbnail
|
||||
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
url=url,
|
||||
title=title,
|
||||
content=" / ".join(x for x in [source, pub_date] if x),
|
||||
thumbnail=thumbnail or "",
|
||||
content=content,
|
||||
thumbnail=thumbnail,
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
return res
|
||||
|
||||
|
||||
ceid_list = [
|
||||
"AE:ar",
|
||||
"AR:es-419",
|
||||
"AT:de",
|
||||
"AU:en",
|
||||
"BD:bn",
|
||||
"BE:fr",
|
||||
"BE:nl",
|
||||
"BG:bg",
|
||||
"BR:pt-419",
|
||||
"BW:en",
|
||||
"CA:en",
|
||||
"CA:fr",
|
||||
"CH:de",
|
||||
"CH:fr",
|
||||
"CL:es-419",
|
||||
"CN:zh-Hans",
|
||||
"CO:es-419",
|
||||
"CU:es-419",
|
||||
"CZ:cs",
|
||||
"DE:de",
|
||||
"EE:et",
|
||||
"EG:ar",
|
||||
"ES:ca",
|
||||
"ES:es",
|
||||
"ET:en",
|
||||
"FI:fi",
|
||||
"FR:fr",
|
||||
"GB:en",
|
||||
"GH:en",
|
||||
"GR:el",
|
||||
"HK:zh-Hant",
|
||||
"HU:hu",
|
||||
"ID:en",
|
||||
"ID:id",
|
||||
"IE:en",
|
||||
"IL:en",
|
||||
"IL:he",
|
||||
"IN:bn",
|
||||
"IN:en",
|
||||
"IN:gu",
|
||||
"IN:hi",
|
||||
"IN:ml",
|
||||
"IN:mr",
|
||||
"IN:pa",
|
||||
"IN:ta",
|
||||
"IN:te",
|
||||
"IT:it",
|
||||
"JP:ja",
|
||||
"KE:en",
|
||||
"KR:ko",
|
||||
"LB:ar",
|
||||
"LT:lt",
|
||||
"LV:en",
|
||||
"LV:lv",
|
||||
"MA:fr",
|
||||
"MY:en",
|
||||
"MY:ms",
|
||||
"NA:en",
|
||||
"NG:en",
|
||||
"NL:nl",
|
||||
"NO:no",
|
||||
"NZ:en",
|
||||
"PH:en",
|
||||
"PK:en",
|
||||
"PL:pl",
|
||||
"RO:ro",
|
||||
"RS:sr",
|
||||
"RU:ru",
|
||||
"SA:ar",
|
||||
"SE:sv",
|
||||
"SG:en",
|
||||
"SI:sl",
|
||||
"SK:sk",
|
||||
"SN:fr",
|
||||
"TH:th",
|
||||
"TR:tr",
|
||||
"TZ:en",
|
||||
"UA:ru",
|
||||
"UA:uk",
|
||||
"UG:en",
|
||||
"US:en",
|
||||
"VN:vi",
|
||||
"ZA:en",
|
||||
"ZW:en",
|
||||
]
|
||||
"""List of region/language combinations supported by Google News. Values of the
|
||||
``ceid`` argument of the Google News REST API."""
|
||||
|
||||
|
||||
_skip_values = [
|
||||
"ET:en", # english (ethiopia)
|
||||
"ID:en", # english (indonesia)
|
||||
"LV:en", # english (latvia)
|
||||
]
|
||||
|
||||
_ceid_locale_map = {"NO:no": "nb-NO"}
|
||||
|
||||
|
||||
def fetch_traits(engine_traits: EngineTraits):
|
||||
_fetch_traits(engine_traits, add_domains=False)
|
||||
|
||||
engine_traits.custom["ceid"] = {}
|
||||
|
||||
for ceid in ceid_list:
|
||||
if ceid in _skip_values:
|
||||
continue
|
||||
|
||||
region, lang = ceid.split(":")
|
||||
x = lang.split("-")
|
||||
if len(x) > 1:
|
||||
if x[1] not in ["Hant", "Hans"]:
|
||||
lang = x[0]
|
||||
|
||||
sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
|
||||
try:
|
||||
locale = babel.Locale.parse(sxng_locale, sep="-")
|
||||
except babel.UnknownLocaleError:
|
||||
print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
|
||||
continue
|
||||
|
||||
engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid
|
||||
|
||||
@@ -20,7 +20,6 @@ about = {
|
||||
}
|
||||
|
||||
play_categ = None # apps|movies
|
||||
enable_http3 = True
|
||||
base_url = 'https://play.google.com'
|
||||
search_url = base_url + "/store/search?{query}&c={play_categ}"
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ import typing as t
|
||||
from urllib.parse import urlencode
|
||||
from datetime import datetime
|
||||
from lxml import html
|
||||
from curl_cffi.requests.exceptions import TooManyRedirects
|
||||
import httpx
|
||||
|
||||
from searx.utils import (
|
||||
eval_xpath,
|
||||
@@ -63,7 +63,6 @@ about = {
|
||||
# engine dependent config
|
||||
categories = ["science", "scientific publications"]
|
||||
paging = True
|
||||
enable_http3 = True
|
||||
max_page = 50
|
||||
"""`Google max 50 pages`_
|
||||
|
||||
@@ -78,6 +77,8 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
"""Google-Scholar search request"""
|
||||
|
||||
google_info = get_google_info(params, traits)
|
||||
# subdomain is: scholar.google.xy
|
||||
google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
|
||||
|
||||
args = {
|
||||
"q": query,
|
||||
@@ -88,7 +89,7 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
}
|
||||
args.update(time_range_args(params))
|
||||
|
||||
params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
|
||||
params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
|
||||
params["cookies"] = google_info["cookies"]
|
||||
params["headers"].update(google_info["headers"])
|
||||
|
||||
@@ -103,7 +104,7 @@ def response(resp: "SXNG_Response") -> EngineResults: # pylint: disable=too-man
|
||||
raise SearxEngineAccessDeniedException(
|
||||
message="google_scholar: unusual traffic detected",
|
||||
)
|
||||
raise TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
|
||||
raise httpx.TooManyRedirects(f"location {resp.headers['Location'].split('?')[0]}")
|
||||
|
||||
res = EngineResults()
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
@@ -1,85 +1,186 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Google Videos: see :py:obj:`searx.engines.google`."""
|
||||
"""This is the implementation of the Google Videos engine.
|
||||
|
||||
import typing as t
|
||||
.. admonition:: Content-Security-Policy (CSP)
|
||||
|
||||
This engine needs to allow images from the `data URLs`_ (prefixed with the
|
||||
``data:`` scheme)::
|
||||
|
||||
Header set Content-Security-Policy "img-src 'self' data: ;"
|
||||
|
||||
.. _data URLs:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
|
||||
"""
|
||||
|
||||
import re
|
||||
from urllib.parse import urlencode, urlparse, parse_qs, unquote
|
||||
from lxml import html
|
||||
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import google_request, unwrap_google_url, wml_dom
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import (
|
||||
eval_xpath_getindex,
|
||||
eval_xpath_list,
|
||||
eval_xpath_getindex,
|
||||
extract_text,
|
||||
parse_duration_string,
|
||||
)
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import (
|
||||
get_google_info,
|
||||
time_range_dict,
|
||||
filter_mapping,
|
||||
suggestion_xpath,
|
||||
detect_google_sorry,
|
||||
ui_async,
|
||||
)
|
||||
from searx.utils import get_embeded_stream_url
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": "https://www.google.com",
|
||||
"wikidata_id": "Q219885",
|
||||
"official_api_documentation": "https://developers.google.com/custom-search",
|
||||
"website": 'https://www.google.com',
|
||||
"wikidata_id": 'Q219885',
|
||||
"official_api_documentation": 'https://developers.google.com/custom-search',
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "XML",
|
||||
"results": 'HTML',
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ["videos", "web"]
|
||||
categories = ['videos', 'web']
|
||||
paging = True
|
||||
max_page = 50
|
||||
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
|
||||
|
||||
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
language_support = True
|
||||
time_range_support = True
|
||||
safesearch = True
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
google_request(
|
||||
query,
|
||||
params,
|
||||
{"tbm": "vid"},
|
||||
eng_traits=traits,
|
||||
use_locales=False,
|
||||
# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
|
||||
# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
|
||||
RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
|
||||
|
||||
|
||||
def parse_data_images(text: str):
|
||||
data_image_map = {}
|
||||
|
||||
for img_id, data_image in RE_DATA_IMAGE.findall(text):
|
||||
end_pos = data_image.rfind("=")
|
||||
if end_pos > 0:
|
||||
data_image = data_image[: end_pos + 1]
|
||||
data_image_map[img_id] = data_image
|
||||
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
|
||||
return data_image_map
|
||||
|
||||
|
||||
def request(query, params):
|
||||
"""Google-Video search request"""
|
||||
google_info = get_google_info(params, traits)
|
||||
start = (params['pageno'] - 1) * 10
|
||||
|
||||
query_url = (
|
||||
'https://'
|
||||
+ google_info['subdomain']
|
||||
+ '/search'
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
'q': query,
|
||||
'tbm': "vid",
|
||||
'start': start,
|
||||
**google_info['params'],
|
||||
'asearch': 'arc',
|
||||
'async': ui_async(start),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if params['time_range'] in time_range_dict:
|
||||
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
|
||||
if 'safesearch' in params:
|
||||
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
|
||||
params['url'] = query_url
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
results = EngineResults()
|
||||
params['cookies'] = google_info['cookies']
|
||||
params['headers'].update(google_info['headers'])
|
||||
return params
|
||||
|
||||
for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
|
||||
|
||||
def response(resp):
|
||||
"""Get response from google's search request"""
|
||||
results = []
|
||||
|
||||
detect_google_sorry(resp)
|
||||
data_image_map = parse_data_images(resp.text)
|
||||
|
||||
# convert the text to dom
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
|
||||
|
||||
# parse results
|
||||
for result in result_divs:
|
||||
title = extract_text(
|
||||
eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
|
||||
eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
|
||||
allow_none=True,
|
||||
)
|
||||
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
|
||||
if not title or not raw_url:
|
||||
continue
|
||||
|
||||
url = unwrap_google_url(raw_url)
|
||||
thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
|
||||
if "/default.jpg" in thumbnail:
|
||||
thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
|
||||
length = None
|
||||
for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
|
||||
length = parse_duration_string(extract_text(span) or "")
|
||||
if length:
|
||||
break
|
||||
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
url=url,
|
||||
title=title,
|
||||
thumbnail=thumbnail,
|
||||
length=length,
|
||||
template="videos.html",
|
||||
)
|
||||
url = eval_xpath_getindex(
|
||||
result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
|
||||
)
|
||||
if url and url.startswith('/url?q='):
|
||||
url = unquote(url[7:].split('&sa=U')[0])
|
||||
|
||||
content = extract_text(
|
||||
eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
|
||||
)
|
||||
pub_info = extract_text(
|
||||
eval_xpath_getindex(
|
||||
result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
|
||||
),
|
||||
allow_none=True,
|
||||
)
|
||||
# Broader XPath to find any <img> element
|
||||
thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
|
||||
duration = extract_text(
|
||||
eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
|
||||
)
|
||||
video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
|
||||
|
||||
# Fallback for video_id from URL if not found via XPath
|
||||
if not video_id and url and 'youtube.com' in url:
|
||||
parsed_url = urlparse(url)
|
||||
video_id = parse_qs(parsed_url.query).get('v', [None])[0]
|
||||
|
||||
# Handle thumbnail
|
||||
if thumbnail and thumbnail.startswith('data:image'):
|
||||
img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
|
||||
if img_id and img_id in data_image_map:
|
||||
thumbnail = data_image_map[img_id]
|
||||
else:
|
||||
thumbnail = None
|
||||
if not thumbnail and video_id:
|
||||
thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
|
||||
|
||||
# Handle video embed URL
|
||||
embed_url = None
|
||||
if video_id:
|
||||
embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
|
||||
elif url:
|
||||
embed_url = get_embeded_stream_url(url)
|
||||
|
||||
# Only append results with valid title and url
|
||||
if title and url:
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content or '',
|
||||
'author': pub_info,
|
||||
'thumbnail': thumbnail,
|
||||
'length': duration,
|
||||
'iframe_src': embed_url,
|
||||
'template': 'videos.html',
|
||||
}
|
||||
)
|
||||
|
||||
# parse suggestion
|
||||
for suggestion in eval_xpath_list(dom, suggestion_xpath):
|
||||
results.append({'suggestion': extract_text(suggestion)})
|
||||
|
||||
return results
|
||||
|
||||
127
searx/engines/heexy.py
Normal file
127
searx/engines/heexy.py
Normal file
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Heexy_ is a minimalist search engine that focuses on privacy.
|
||||
|
||||
Although it also supports news and videos, these are not implemented here
|
||||
because they usually return no result to very few irrelevant ones.
|
||||
|
||||
It seems to use Bing internally, as the image thumbnails are loaded from Bing.
|
||||
|
||||
.. _Heexy: https://docs.heexy.org/introduction
|
||||
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import typing as t
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.network import get
|
||||
from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import gen_useragent
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
about = {
|
||||
"website": "https://heexy.org",
|
||||
"wikidata_id": None,
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
paging = True
|
||||
safesearch = True
|
||||
|
||||
categories = ["general"]
|
||||
heexy_categ = "web"
|
||||
"""Category to search in. Can be either "web" or "image"."""
|
||||
|
||||
|
||||
base_url = "https://heexy.org"
|
||||
api_url = "https://seapi.heexy.org"
|
||||
safe_search_map = {0: "off", 1: "on", 2: "on"}
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache for storing the ``X-Data-Cacheft`` token (acts like an API key)."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
|
||||
if heexy_categ not in ("web", "image"):
|
||||
raise ValueError("invalid search category: %s" % heexy_categ)
|
||||
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
|
||||
|
||||
def _get_api_token(query: str) -> str:
|
||||
"""The API token is independent of the search query. We just need any query
|
||||
to obtain it initially, and don't hardcode it here to decrease chances of
|
||||
getting blocked. The token must be passed as ``X-Data-Cacheft`` header."""
|
||||
|
||||
cached_token: str = CACHE.get("token")
|
||||
if cached_token:
|
||||
return cached_token
|
||||
|
||||
resp = get(
|
||||
f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent(), "Accept-Language": "en-US,en:q=0.9"}
|
||||
)
|
||||
if not resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain request token: invalid response code")
|
||||
|
||||
token = resp.cookies["cacheft"]
|
||||
if not token:
|
||||
raise SearxEngineAPIException("failed to obtain request token: no token found")
|
||||
|
||||
CACHE.set("token", token, expire=3 * 60)
|
||||
return token
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
args = {
|
||||
"q": query,
|
||||
"page": params["pageno"],
|
||||
"safe": safe_search_map[params["safesearch"]],
|
||||
}
|
||||
if params["searxng_locale"] != "all":
|
||||
args["lang"] = params["searxng_locale"].split("-")[0]
|
||||
|
||||
params["url"] = f"{api_url}/search/{heexy_categ}?{urlencode(args)}"
|
||||
|
||||
params["headers"]["Origin"] = api_url
|
||||
params["cookies"]["cacheft"] = _get_api_token(query)
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
res = EngineResults()
|
||||
|
||||
json_resp = resp.json()
|
||||
if not json_resp["success"]:
|
||||
raise SearxEngineAccessDeniedException()
|
||||
|
||||
result: dict[str, str]
|
||||
for result in json_resp["results"]:
|
||||
if heexy_categ == "web":
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
url=result["url"],
|
||||
title=result["title"],
|
||||
content=result["description"],
|
||||
)
|
||||
)
|
||||
elif heexy_categ == "image":
|
||||
res.add(
|
||||
res.types.Image(
|
||||
title=result["description"],
|
||||
url=result["url"],
|
||||
thumbnail_src=result["image"],
|
||||
img_src=result["rawImage"],
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
@@ -4,7 +4,6 @@
|
||||
from urllib.parse import urlencode
|
||||
from dateutil import parser
|
||||
|
||||
|
||||
about = {
|
||||
# pylint: disable=line-too-long
|
||||
"website": "https://hex.pm/",
|
||||
|
||||
@@ -33,7 +33,6 @@ Implementations
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
from urllib.parse import urlencode
|
||||
from datetime import datetime
|
||||
|
||||
@@ -63,7 +62,7 @@ huggingface_endpoint = 'models'
|
||||
"""
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if huggingface_endpoint not in ('datasets', 'models', 'spaces'):
|
||||
raise SearxEngineAPIException(f"Unsupported Hugging Face endpoint: {huggingface_endpoint}")
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ No public instance offer a public API now
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
import time
|
||||
import random
|
||||
from urllib.parse import quote_plus, urlparse
|
||||
@@ -35,7 +34,7 @@ time_range_support = True
|
||||
base_url: list[str] | str = []
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if not base_url:
|
||||
raise ValueError("missing invidious base_url")
|
||||
|
||||
|
||||
@@ -108,14 +108,12 @@ def get_infobox(alt_forms, result_url, definitions):
|
||||
infobox_content.append(f'<p><i>Other forms:</i> {", ".join(alt_forms[1:])}</p>')
|
||||
|
||||
# definitions
|
||||
infobox_content.append(
|
||||
'''
|
||||
infobox_content.append('''
|
||||
<small><a href="https://www.edrdg.org/wiki/index.php/JMdict-EDICT_Dictionary_Project">JMdict</a>
|
||||
and <a href="https://www.edrdg.org/enamdict/enamdict_doc.html">JMnedict</a>
|
||||
by <a href="https://www.edrdg.org/edrdg/licence.html">EDRDG</a>, CC BY-SA 3.0.</small>
|
||||
<ul>
|
||||
'''
|
||||
)
|
||||
''')
|
||||
for pos, engdef, extra in definitions:
|
||||
if pos == 'Wikipedia definition':
|
||||
infobox_content.append('</ul><small>Wikipedia, CC BY-SA 3.0.</small><ul>')
|
||||
|
||||
@@ -87,7 +87,7 @@ api_key = ""
|
||||
"""Kagi API key. Required for using this engine."""
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if not api_key:
|
||||
raise ValueError("api_key is required for using kagi")
|
||||
|
||||
|
||||
@@ -37,17 +37,21 @@ about = {
|
||||
}
|
||||
|
||||
categories = []
|
||||
safesearch = True
|
||||
safeseach = True
|
||||
|
||||
base_url = "https://luxxle.com"
|
||||
|
||||
luxxle_categ = "search"
|
||||
"""Supported categories: "search", "news", "images", "videos"."""
|
||||
|
||||
# otherwise all requests get blocked (http2-fingerprinted probably)
|
||||
enable_http2 = False
|
||||
|
||||
|
||||
safe_search_map = {0: "Off", 1: "Moderate", 2: "Strict"}
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if luxxle_categ not in ("search", "images", "videos", "news"):
|
||||
raise ValueError("invalid luxxle category: %s" % luxxle_categ)
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ Lofgren .
|
||||
.. _marginalia filters:
|
||||
|
||||
Marginalia Filters
|
||||
==================
|
||||
=================
|
||||
|
||||
Custom filters enable server-side customization of Marginalia search results.
|
||||
Filter definitions are written in XML and scoped to an API key. Filters can
|
||||
@@ -82,7 +82,7 @@ api_key = None
|
||||
https://about.marginalia-search.com/article/api/
|
||||
|
||||
"""
|
||||
filter_name: str = ""
|
||||
filter_name: str | None = None
|
||||
"""The name of the custom filter to apply to each search."""
|
||||
|
||||
|
||||
@@ -164,7 +164,7 @@ def response(resp: SXNG_Response):
|
||||
return res
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool:
|
||||
def init(_: dict[str, t.Any]):
|
||||
|
||||
if not api_key:
|
||||
logger.error("missing api_key: see https://about.marginalia-search.com/article/api")
|
||||
|
||||
@@ -26,8 +26,6 @@ Implementations
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
try:
|
||||
import mariadb # pyright: ignore [reportMissingImports]
|
||||
except ImportError:
|
||||
@@ -62,17 +60,15 @@ paging = True
|
||||
_connection = None
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
|
||||
def init(engine_settings):
|
||||
global _connection # pylint: disable=global-statement
|
||||
|
||||
if 'query_str' not in engine_settings:
|
||||
raise ValueError('query_str cannot be empty')
|
||||
|
||||
if not engine_settings['query_str'].lower().startswith('select '):
|
||||
raise ValueError('only SELECT query is supported')
|
||||
|
||||
|
||||
def init(_):
|
||||
global _connection # pylint: disable=global-statement
|
||||
|
||||
_connection = mariadb.connect(database=database, user=username, password=password, host=host, port=port)
|
||||
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ Here is a simple example to query a Meilisearch instance:
|
||||
"""
|
||||
|
||||
# pylint: disable=global-statement
|
||||
import typing as t
|
||||
|
||||
from json import dumps
|
||||
from searx.result_types import EngineResults
|
||||
from searx.extended_types import SXNG_Response
|
||||
@@ -49,7 +49,7 @@ categories = ['general']
|
||||
paging = True
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if index == '':
|
||||
raise ValueError('index cannot be empty')
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Mojeek (general, images, news)"""
|
||||
|
||||
import typing as t
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlencode
|
||||
|
||||
@@ -52,7 +51,7 @@ region_param = "arc"
|
||||
_delta_kwargs = {"day": "days", "week": "weeks", "month": "months", "year": "years"}
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if search_type not in ("", "images", "news"):
|
||||
raise ValueError(f"Invalid search type {search_type}")
|
||||
|
||||
|
||||
@@ -49,7 +49,6 @@ except ImportError:
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
|
||||
engine_type = 'offline'
|
||||
|
||||
# mongodb connection variables
|
||||
|
||||
@@ -19,7 +19,6 @@ Implementation
|
||||
==============
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
about = {
|
||||
@@ -38,7 +37,7 @@ matrix_url = "https://matrix.to"
|
||||
page_size = 20
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(engine_settings): # pylint: disable=unused-argument
|
||||
"""The ``base_url`` must be set in the configuration, if ``base_url`` is not
|
||||
set, a :py:obj:`ValueError` is raised during initialization.
|
||||
|
||||
|
||||
@@ -25,8 +25,6 @@ Implementations
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
try:
|
||||
@@ -62,17 +60,15 @@ paging = True
|
||||
_connection = None
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool | None:
|
||||
def init(engine_settings):
|
||||
global _connection # pylint: disable=global-statement
|
||||
|
||||
if 'query_str' not in engine_settings:
|
||||
raise ValueError('query_str cannot be empty')
|
||||
|
||||
if not engine_settings['query_str'].lower().startswith('select '):
|
||||
raise ValueError('only SELECT query is supported')
|
||||
|
||||
|
||||
def init(_):
|
||||
global _connection # pylint: disable=global-statement
|
||||
|
||||
_connection = mysql.connector.connect(
|
||||
database=database,
|
||||
user=username,
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# pylint: disable=line-too-long
|
||||
"""Naver for SearXNG"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from urllib.parse import urlencode
|
||||
from lxml import html
|
||||
|
||||
@@ -18,6 +16,7 @@ from searx.utils import (
|
||||
html_to_text,
|
||||
parse_duration_string,
|
||||
js_obj_str_to_python,
|
||||
get_embeded_stream_url,
|
||||
)
|
||||
|
||||
# engine metadata
|
||||
@@ -68,7 +67,7 @@ naver_category_dict = {
|
||||
}
|
||||
|
||||
|
||||
def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
def init(_):
|
||||
if naver_category not in ('general', 'images', 'news', 'videos'):
|
||||
raise SearxEngineAPIException(f"Unsupported category: {naver_category}")
|
||||
|
||||
@@ -194,7 +193,7 @@ def parse_news(data):
|
||||
|
||||
|
||||
def parse_videos(data):
|
||||
res = EngineResults()
|
||||
results = []
|
||||
|
||||
dom = html.fromstring(data)
|
||||
|
||||
@@ -213,14 +212,15 @@ def parse_videos(data):
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
res.add(
|
||||
res.types.LegacyResult(
|
||||
template="videos.html",
|
||||
title=extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
|
||||
url=url,
|
||||
thumbnail=thumbnail,
|
||||
length=length,
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"template": "videos.html",
|
||||
"title": extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
|
||||
"url": url,
|
||||
"thumbnail": thumbnail,
|
||||
"length": length,
|
||||
"iframe_src": get_embeded_stream_url(url),
|
||||
}
|
||||
)
|
||||
|
||||
return res
|
||||
return results
|
||||
|
||||
@@ -6,14 +6,10 @@
|
||||
|
||||
from json import loads
|
||||
import typing as t
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from lxml import html
|
||||
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.network import get
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import eval_xpath, extract_text
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
@@ -29,33 +25,18 @@ about = {
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
paging = False
|
||||
enable_http3 = True
|
||||
|
||||
base_url = "https://neosearch.org"
|
||||
categories = ["general"]
|
||||
|
||||
|
||||
def _obtain_xsrf_token() -> str:
|
||||
resp = get(base_url)
|
||||
doc = html.fromstring(resp.text)
|
||||
|
||||
xsrf_token = extract_text(eval_xpath(doc, "//meta[@name='xsrf-token']/@content"))
|
||||
if not xsrf_token:
|
||||
raise SearxEngineAPIException("failed to obtain xsrf token")
|
||||
return xsrf_token
|
||||
paging = False
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams"):
|
||||
params["url"] = f"{base_url}/search"
|
||||
params["headers"]["X-XSRF-TOKEN"] = _obtain_xsrf_token()
|
||||
params["method"] = "POST"
|
||||
|
||||
args = {"q": query, "generate": "auto"}
|
||||
countrycode = params["searxng_locale"].split("-")[-1].upper()
|
||||
if countrycode in traits.custom["countrycodes"]:
|
||||
args["loc"] = countrycode
|
||||
params["json"] = args
|
||||
params["url"] = f"{base_url}/search?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
@@ -86,6 +67,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
|
||||
def fetch_traits(engine_traits: "EngineTraits") -> None:
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from searx.network import get
|
||||
from searx.utils import extr, js_obj_str_to_python
|
||||
from babel.core import get_global
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user