mirror of
https://github.com/searxng/searxng.git
synced 2026-09-11 16:56:05 +00:00
Compare commits
14 Commits
00eaf1157d
...
dependabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
401d7f9f51 | ||
|
|
aee33412aa | ||
|
|
79c8ffe0da | ||
|
|
2b1c88c54c | ||
|
|
d226b78bc4 | ||
|
|
bdbf9774f5 | ||
|
|
451c46aa32 | ||
|
|
a30b2d4749 | ||
|
|
9fea41204f | ||
|
|
777ba8fa48 | ||
|
|
a4cb7df053 | ||
|
|
bbb3c7d829 | ||
|
|
f31ff05db3 | ||
|
|
5f4005996e |
2
.github/workflows/data-update.yml
vendored
2
.github/workflows/data-update.yml
vendored
@@ -31,7 +31,7 @@ jobs:
|
||||
- update_external_bangs.py
|
||||
- update_firefox_version.py
|
||||
- update_engine_traits.py
|
||||
- update_wikidata_units.py
|
||||
- update_wikidata.py
|
||||
- update_engine_descriptions.py
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
/*
|
||||
this file is generated automatically by searxng_extra/update/update_pygments.py
|
||||
using pygments version 2.20.0:
|
||||
using pygments version 2.21.0:
|
||||
|
||||
./manage templates.simple.pygments
|
||||
*/
|
||||
@@ -114,14 +114,14 @@
|
||||
.gd { color: #FF4689 } /* Generic.Deleted */
|
||||
.ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
|
||||
.ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
|
||||
.gr { color: #F8F8F2 } /* Generic.Error */
|
||||
.gr { color: #FF4689 } /* Generic.Error */
|
||||
.gh { color: #F8F8F2 } /* Generic.Heading */
|
||||
.gi { color: #A6E22E } /* Generic.Inserted */
|
||||
.go { color: #66D9EF } /* Generic.Output */
|
||||
.gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
|
||||
.gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
|
||||
.gu { color: #959077 } /* Generic.Subheading */
|
||||
.gt { color: #F8F8F2 } /* Generic.Traceback */
|
||||
.gt { color: #66D9EF } /* Generic.Traceback */
|
||||
.kc { color: #66D9EF } /* Keyword.Constant */
|
||||
.kd { color: #66D9EF } /* Keyword.Declaration */
|
||||
.kn { color: #FF4689 } /* Keyword.Namespace */
|
||||
@@ -132,7 +132,7 @@
|
||||
.m { color: #AE81FF } /* Literal.Number */
|
||||
.s { color: #E6DB74 } /* Literal.String */
|
||||
.na { color: #A6E22E } /* Name.Attribute */
|
||||
.nb { color: #F8F8F2 } /* Name.Builtin */
|
||||
.nb { color: #A6E22E } /* Name.Builtin */
|
||||
.nc { color: #A6E22E } /* Name.Class */
|
||||
.no { color: #66D9EF } /* Name.Constant */
|
||||
.nd { color: #A6E22E } /* Name.Decorator */
|
||||
@@ -166,7 +166,7 @@
|
||||
.sr { color: #E6DB74 } /* Literal.String.Regex */
|
||||
.s1 { color: #E6DB74 } /* Literal.String.Single */
|
||||
.ss { color: #E6DB74 } /* Literal.String.Symbol */
|
||||
.bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
|
||||
.bp { color: #A6E22E } /* Name.Builtin.Pseudo */
|
||||
.fm { color: #A6E22E } /* Name.Function.Magic */
|
||||
.vc { color: #F8F8F2 } /* Name.Variable.Class */
|
||||
.vg { color: #F8F8F2 } /* Name.Variable.Global */
|
||||
|
||||
80
client/simple/package-lock.json
generated
80
client/simple/package-lock.json
generated
@@ -15,12 +15,12 @@
|
||||
"swiped-events": "1.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.5.7",
|
||||
"@biomejs/biome": "2.5.9",
|
||||
"@types/node": "^26.2.0",
|
||||
"browserslist": "^4.28.8",
|
||||
"browserslist-to-esbuild": "^2.1.1",
|
||||
"edge.js": "^6.5.1",
|
||||
"less": "^4.8.1",
|
||||
"less": "^4.9.0",
|
||||
"mathjs": "^15.2.0",
|
||||
"sharp": "~0.35.3",
|
||||
"sort-package-json": "^4.0.0",
|
||||
@@ -69,9 +69,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/biome": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.7.tgz",
|
||||
"integrity": "sha512-zr8K/DcY5tYsQOQwqMJ0AWElo6QgmgNI7idXgXLhevVszlt8RGVpesEJPqx3ThazLaOwjJ5Y8fz3BtH5fGZNsw==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.9.tgz",
|
||||
"integrity": "sha512-KkgCvdHB4IhtpHpF564plA9jo6fDOwWGQ/3jvreLzgOtRLEDoPqr7QO9qejNA8jKwDsSkAKr77hqBHnyUbIw4g==",
|
||||
"dev": true,
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"bin": {
|
||||
@@ -85,20 +85,20 @@
|
||||
"url": "https://opencollective.com/biome"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@biomejs/cli-darwin-arm64": "2.5.7",
|
||||
"@biomejs/cli-darwin-x64": "2.5.7",
|
||||
"@biomejs/cli-linux-arm64": "2.5.7",
|
||||
"@biomejs/cli-linux-arm64-musl": "2.5.7",
|
||||
"@biomejs/cli-linux-x64": "2.5.7",
|
||||
"@biomejs/cli-linux-x64-musl": "2.5.7",
|
||||
"@biomejs/cli-win32-arm64": "2.5.7",
|
||||
"@biomejs/cli-win32-x64": "2.5.7"
|
||||
"@biomejs/cli-darwin-arm64": "2.5.9",
|
||||
"@biomejs/cli-darwin-x64": "2.5.9",
|
||||
"@biomejs/cli-linux-arm64": "2.5.9",
|
||||
"@biomejs/cli-linux-arm64-musl": "2.5.9",
|
||||
"@biomejs/cli-linux-x64": "2.5.9",
|
||||
"@biomejs/cli-linux-x64-musl": "2.5.9",
|
||||
"@biomejs/cli-win32-arm64": "2.5.9",
|
||||
"@biomejs/cli-win32-x64": "2.5.9"
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-darwin-arm64": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.7.tgz",
|
||||
"integrity": "sha512-vxo/Ls3/PYdQWyLhYYcgMOCzQypAjcY+iihS8M0wW03l16TCLW4zqZzGo75gm1VdCMj38hTVZ31KBWrZ4G9dJw==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.9.tgz",
|
||||
"integrity": "sha512-am22pX2aBqznqq1eMyIj/bZ++riF3Lk6ct7cbv+gQK0csFhr+d8O0RkOi2FF2qSgFgANbqNkIZ0/PxlnW2pLFg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -113,9 +113,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-darwin-x64": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.7.tgz",
|
||||
"integrity": "sha512-Cd3Ga61amT/Yl/0x8elP5hhGYaFy4bw6WuysTgf7oo8TA5tJ5A1k+DkVoJ2BHbTVil51gTX9VPzArnrlLJ3Kyg==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.9.tgz",
|
||||
"integrity": "sha512-l44KWDHLDvEnD0N/XcrVs7VXb3A18xL7QS3WB0eL93wbmk529ffIG55vleGCqaunpRUjLrdnjK05Qki1dsjylg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -130,9 +130,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-linux-arm64": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.7.tgz",
|
||||
"integrity": "sha512-rR2QE0yF2GYSuYuKIa7pKvODGJqnOH+2eDREAM8wV+mWKSkMQKdAp4zXEZfTaxY8PMoNONnpgSWcBCyLDPDOKg==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.9.tgz",
|
||||
"integrity": "sha512-ICaK+IYaVZvKbBxX2rwrPT0DdUDMnE9Vm3nQGe+mltQPmUg19pONzkPWGdY4FCsoreDETWDynvdt4ysCbF5gNQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -150,9 +150,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-linux-arm64-musl": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.7.tgz",
|
||||
"integrity": "sha512-xPI5yB6XlpDbNkS+bm1t42olw5c4l3UrlOmLg7KtLJvjvkNF/1V4tnUgfkylGIeb3u/T+BzMGYqgQhzjAoJzuQ==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.9.tgz",
|
||||
"integrity": "sha512-7ImVPwBLCtkmpR5esd8RHhTqW94f0JLJQum6AneYcy94jRm18TaPPm7slaigGzFhfgt3QiD1Vj52LKmBAnKizA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -170,9 +170,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-linux-x64": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.7.tgz",
|
||||
"integrity": "sha512-FQgqJhscrqJUFptGaRSUJWlXAExwWcDwLuK49dvKfkQ1bB5SEEyFssnsxQY83Xm6jR0EbbX3+8+D5bfvYqUG2Q==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.9.tgz",
|
||||
"integrity": "sha512-z22Q/zFYSvbIJfW1CbfZPu4X8PddS6Qd2ORbc6h+aT6EcwAxUF3m6fA4HjNvA3TU4X0dTJRwNPB165ES3PJXzg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -190,9 +190,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-linux-x64-musl": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.7.tgz",
|
||||
"integrity": "sha512-rE5VZi+qtmPgQH+l7jVxYoZ18b/TiHEhulhMpjmCZH1PltSbjRcxNWywC3HZ9tYottG7ORkeTtoscBilKSBm0g==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.9.tgz",
|
||||
"integrity": "sha512-RXGaD0o1/pTTguYw1aeDJh9ad6Lfrui0fI7mBderTyGr7WuUJkBIttgLkR3XJyoxOkkgfBDspaUT8wXArTqLZw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -210,9 +210,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-win32-arm64": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.7.tgz",
|
||||
"integrity": "sha512-Oq4x0CCwP4jirrcTywXs5kOGZ4v5vuEP+gWrbtjApOA2CL9F3F9GlIdQIci8AKSCa/zURanMRpX/4wQ7Am6hHg==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.9.tgz",
|
||||
"integrity": "sha512-nHK+/HHC+D0ogAHUxomgoSTdjImb6fmNNVTKmf0tyu4eDL1DqPKIHc+i+UL8+b0RnAu8224qo8F2tCVnaT0A3w==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -227,9 +227,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-win32-x64": {
|
||||
"version": "2.5.7",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.7.tgz",
|
||||
"integrity": "sha512-V+0wu/nrj2S+MhP4EQ0uHNolP0IALEsz45pg0WoKkHfDeh0+ItHwP/p7bX5RPoMOl9NkpHYWdYPhIcy2mACHvQ==",
|
||||
"version": "2.5.9",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.9.tgz",
|
||||
"integrity": "sha512-Yiq0H56LjXSSw/hd9YkXgSLQfzyDJzbzU2TezozxyNw+uKWAqOtqGVvBfzKRRDiaFF5avGAhHdWKx7LtDOShUw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -3233,9 +3233,9 @@
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/less": {
|
||||
"version": "4.8.1",
|
||||
"resolved": "https://registry.npmjs.org/less/-/less-4.8.1.tgz",
|
||||
"integrity": "sha512-jQ3lRIo1aUtiWVYXZ7mk4+V4BjCGswF3IxTLJ+4RUta8ZiHh8lhkig2G8dya2eCcyR1dYUvzuV46EkJN8PSwww==",
|
||||
"version": "4.9.0",
|
||||
"resolved": "https://registry.npmjs.org/less/-/less-4.9.0.tgz",
|
||||
"integrity": "sha512-umRhrCH7fCi8Uj2RcwKjJdvUORTjeWqkdKx0LbcZvjIwsAVsnIAGcxHaqowPeBFBjQuWOeC/bve0AlpFzF/+SQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
|
||||
@@ -29,12 +29,12 @@
|
||||
"swiped-events": "1.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.5.7",
|
||||
"@biomejs/biome": "2.5.9",
|
||||
"@types/node": "^26.2.0",
|
||||
"browserslist": "^4.28.8",
|
||||
"browserslist-to-esbuild": "^2.1.1",
|
||||
"edge.js": "^6.5.1",
|
||||
"less": "^4.8.1",
|
||||
"less": "^4.9.0",
|
||||
"mathjs": "^15.2.0",
|
||||
"sharp": "~0.35.3",
|
||||
"sort-package-json": "^4.0.0",
|
||||
|
||||
@@ -80,8 +80,8 @@ same environment, here are a few examples::
|
||||
# to test one of the update scripts
|
||||
(dev.env)$ searxng_extra/update/update_engine_traits.py --help
|
||||
|
||||
# to test the update of the wikidata units
|
||||
(dev.env)$ searxng_extra/update/update_wikidata_units.py
|
||||
# to test the update of the wikidata units and property names
|
||||
(dev.env)$ searxng_extra/update/update_wikidata.py
|
||||
|
||||
|
||||
.. sidebar:: further read
|
||||
|
||||
@@ -90,10 +90,10 @@ Scripts to update static data in :origin:`searx/data/`
|
||||
:members:
|
||||
|
||||
|
||||
``update_wikidata_units.py``
|
||||
``update_wikidata.py``
|
||||
============================
|
||||
|
||||
:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
|
||||
:origin:`[source] <searxng_extra/update/update_wikidata.py>`
|
||||
|
||||
.. automodule:: searxng_extra.update.update_wikidata_units
|
||||
.. automodule:: searxng_extra.update.update_wikidata
|
||||
:members:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
mock==5.2.0
|
||||
nose2[coverage_plugin]==0.16.0
|
||||
cov-core==1.15.0
|
||||
black==25.9.0
|
||||
black==26.5.1
|
||||
pylint==4.0.7
|
||||
splinter==0.21.0
|
||||
selenium==4.47.0
|
||||
@@ -23,6 +23,6 @@ coloredlogs==15.0.1
|
||||
docutils>=0.21.2;python_version <= "3.11"
|
||||
docutils>=0.22.4; python_version > "3.11"
|
||||
parameterized==0.9.0
|
||||
granian[reload]==2.8.1
|
||||
basedpyright==1.39.9
|
||||
granian[reload]==2.8.2
|
||||
basedpyright==1.39.10
|
||||
types-lxml==2026.2.16
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
granian==2.8.1
|
||||
granian[pname]==2.8.1
|
||||
granian==2.8.2
|
||||
granian[pname]==2.8.2
|
||||
|
||||
@@ -3,8 +3,8 @@ babel==2.18.0
|
||||
flask-babel==4.0.0
|
||||
flask==3.1.3
|
||||
jinja2==3.1.6
|
||||
lxml==6.1.1
|
||||
pygments==2.20.0
|
||||
lxml==6.1.2
|
||||
pygments==2.21.0
|
||||
python-dateutil==2.9.0.post0
|
||||
pyyaml==6.0.3
|
||||
httpx[http2]==0.28.1
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Implementation of the :py:obj:`preference <searx.preference>` settings."""
|
||||
|
||||
# pylint: disable = too-few-public-methods
|
||||
|
||||
import typing as t
|
||||
|
||||
@@ -38,7 +38,6 @@ area:
|
||||
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["AnswererInfo", "Answerer", "AnswerStorage"]
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ from dataclasses import dataclass
|
||||
from searx.utils import load_module
|
||||
from searx.result_types.answer import BaseAnswer
|
||||
|
||||
|
||||
_default = pathlib.Path(__file__).parent
|
||||
log: logging.Logger = logging.getLogger("searx.answerers")
|
||||
|
||||
|
||||
@@ -127,18 +127,17 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
|
||||
|
||||
|
||||
def google_complete(query: str, sxng_locale: str) -> list[str]:
|
||||
"""Autocomplete from Google. Supports Google's languages and subdomains
|
||||
"""Autocomplete from Google. Supports Google's languages
|
||||
(:py:obj:`searx.engines.google.get_google_info`) by using the async REST
|
||||
API::
|
||||
|
||||
https://{subdomain}/complete/search?{args}
|
||||
https://www.google.com/complete/search?{args}
|
||||
|
||||
"""
|
||||
|
||||
data = ENGINE_TRAITS.get("google") or {}
|
||||
traits = EngineTraits(**data)
|
||||
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
|
||||
url = 'https://{subdomain}/complete/search?{args}'
|
||||
args = urlencode(
|
||||
{
|
||||
'q': query,
|
||||
@@ -148,7 +147,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
|
||||
)
|
||||
results: list[str] = []
|
||||
|
||||
resp = get(url.format(subdomain=google_info['subdomain'], args=args))
|
||||
resp = get('https://www.google.com/complete/search?' + args)
|
||||
if resp and resp.ok:
|
||||
json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
|
||||
data = json.loads(json_txt)
|
||||
|
||||
@@ -5,7 +5,6 @@ Implementations used for bot detection.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["init", "dump_request", "get_network", "too_many_requests", "ProxyFix"]
|
||||
|
||||
|
||||
|
||||
@@ -182,7 +182,7 @@ class Config:
|
||||
if default is UNSET:
|
||||
raise KeyError(name)
|
||||
return default
|
||||
(modulename, name) = str(fqn).rsplit('.', 1)
|
||||
modulename, name = str(fqn).rsplit('.', 1)
|
||||
m = __import__(modulename, {}, {}, [name], 0)
|
||||
return getattr(m, name)
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ Accept_ header ..
|
||||
|
||||
"""
|
||||
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
|
||||
@@ -14,7 +14,6 @@ bot if the Accept-Encoding_ header ..
|
||||
|
||||
"""
|
||||
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
|
||||
@@ -11,7 +11,6 @@ if the Accept-Language_ header is unset.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
|
||||
@@ -11,7 +11,6 @@ the Connection_ header is set to ``close``.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
|
||||
@@ -20,6 +20,7 @@ Metadata`_. A request is filtered out in case of:
|
||||
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ the User-Agent_ header is unset or matches the regular expression
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import re
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
@@ -25,7 +24,6 @@ import flask
|
||||
from . import config
|
||||
from ._helpers import too_many_requests
|
||||
|
||||
|
||||
USER_AGENT = (
|
||||
r'('
|
||||
+ r'unknown'
|
||||
|
||||
@@ -55,7 +55,6 @@ from ._helpers import (
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
logger = logger.getChild('ip_limit')
|
||||
|
||||
BURST_WINDOW = 20
|
||||
|
||||
@@ -23,6 +23,7 @@ The ``ip_lists`` method implements :py:obj:`block-list <block_ip>` and
|
||||
]
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Implementation of a middleware to determine the real IP of an HTTP request
|
||||
(:py:obj:`flask.request.remote_addr`) behind a proxy chain."""
|
||||
|
||||
# pylint: disable=too-many-branches
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Providing a Valkey database for the botdetection methods."""
|
||||
|
||||
|
||||
import valkey
|
||||
|
||||
__all__ = ["set_valkey_client", "get_valkey_client"]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Implementations needed for a branding of SearXNG."""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
|
||||
# Struct fields aren't discovered in Python 3.14
|
||||
|
||||
@@ -465,7 +465,7 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
|
||||
|
||||
# Check if value is expired. It's possible that it's expired but has not
|
||||
# yet been automatically deleted by the periodic maintenance
|
||||
(value, expire) = row
|
||||
value, expire = row
|
||||
now = time.time()
|
||||
if expire < now:
|
||||
# The record is deleted during the maintenance interval. Deleting
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import warnings
|
||||
|
||||
|
||||
# limiter backward compatibility
|
||||
# ------------------------------
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
make data.all
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
|
||||
__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
|
||||
@@ -32,6 +33,13 @@ class WikiDataUnitType(t.TypedDict):
|
||||
to_si_factor: float
|
||||
|
||||
|
||||
WikiDataPropertyNameType = str | dict[str, str]
|
||||
"""Name of a Wikidata property. Can be either the plain name or a dictionary of
|
||||
language code to property name, e.g. ``{"en": "Date of birth"}``."""
|
||||
WikiDataPropertiesType = dict[str, WikiDataPropertyNameType]
|
||||
"""Dictionary from wikidata property ID to property name."""
|
||||
|
||||
|
||||
class LocalesType(t.TypedDict):
|
||||
"""Data structure of an item in ``locales.json``"""
|
||||
|
||||
@@ -41,6 +49,7 @@ class LocalesType(t.TypedDict):
|
||||
|
||||
USER_AGENTS: UserAgentType
|
||||
WIKIDATA_UNITS: dict[str, WikiDataUnitType]
|
||||
WIKIDATA_PROPERTIES: WikiDataPropertiesType
|
||||
TRACKER_PATTERNS: TrackerPatternsDB
|
||||
LOCALES: LocalesType
|
||||
CURRENCIES: CurrenciesDB
|
||||
@@ -52,11 +61,12 @@ ENGINE_DESCRIPTIONS: dict[str, dict[str, t.Any]]
|
||||
ENGINE_TRAITS: dict[str, dict[str, t.Any]]
|
||||
|
||||
|
||||
lazy_globals = {
|
||||
lazy_globals: dict[str, t.Any] = {
|
||||
"CURRENCIES": CurrenciesDB(),
|
||||
"USER_AGENTS": None,
|
||||
"EXTERNAL_URLS": None,
|
||||
"WIKIDATA_UNITS": None,
|
||||
"WIKIDATA_PROPERTIES": None,
|
||||
"EXTERNAL_BANGS": None,
|
||||
"OSM_KEYS_TAGS": None,
|
||||
"ENGINE_DESCRIPTIONS": None,
|
||||
@@ -69,6 +79,7 @@ data_json_files = {
|
||||
"USER_AGENTS": "useragents.json",
|
||||
"EXTERNAL_URLS": "external_urls.json",
|
||||
"WIKIDATA_UNITS": "wikidata_units.json",
|
||||
"WIKIDATA_PROPERTIES": "wikidata_properties.json",
|
||||
"EXTERNAL_BANGS": "external_bangs.json",
|
||||
"OSM_KEYS_TAGS": "osm_keys_tags.json",
|
||||
"ENGINE_DESCRIPTIONS": "engine_descriptions.json",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Simple implementation to store TrackerPatterns data in a SQL database."""
|
||||
|
||||
# pylint: disable=too-many-branches
|
||||
|
||||
import typing as t
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
],
|
||||
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
|
||||
"versions": [
|
||||
"153.0",
|
||||
"152.0"
|
||||
"154.0",
|
||||
"153.0"
|
||||
]
|
||||
}
|
||||
23618
searx/data/wikidata_properties.json
Normal file
23618
searx/data/wikidata_properties.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -3474,11 +3474,6 @@
|
||||
"symbol": "mm⁻²",
|
||||
"to_si_factor": 1e-06
|
||||
},
|
||||
"Q136039973": {
|
||||
"si_name": "Q6137407",
|
||||
"symbol": "FPS",
|
||||
"to_si_factor": 1.0
|
||||
},
|
||||
"Q1361854": {
|
||||
"si_name": "Q11570",
|
||||
"symbol": "dwt",
|
||||
@@ -5254,6 +5249,11 @@
|
||||
"symbol": "μA",
|
||||
"to_si_factor": 1e-06
|
||||
},
|
||||
"Q31274648": {
|
||||
"si_name": "Q6137407",
|
||||
"symbol": "FPS",
|
||||
"to_si_factor": 1.0
|
||||
},
|
||||
"Q3186734": {
|
||||
"si_name": "Q3186734",
|
||||
"symbol": "J/(m³ K)",
|
||||
|
||||
@@ -25,6 +25,7 @@ To use this engine, add an entry similar to the following to your engine list in
|
||||
https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""BASE (Scholar publications)"""
|
||||
|
||||
from datetime import datetime
|
||||
import re
|
||||
|
||||
|
||||
@@ -83,7 +83,6 @@ from threading import Thread
|
||||
from searx import logger
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
|
||||
engine_type = 'offline'
|
||||
paging = True
|
||||
command = []
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Docker Hub (IT)"""
|
||||
|
||||
# pylint: disable=use-dict-literal
|
||||
|
||||
from urllib.parse import urlencode
|
||||
|
||||
@@ -164,6 +164,7 @@ Terms / phrases that you keep coming across:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Accept-Language
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=global-statement
|
||||
|
||||
import json
|
||||
|
||||
@@ -12,6 +12,7 @@ least we could not find out how language support should work. It seems that
|
||||
most of the features are based on English terms.
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from urllib.parse import urlencode, urlparse, urljoin
|
||||
|
||||
@@ -17,7 +17,6 @@ from searx.result_types import EngineResults
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx import weather
|
||||
|
||||
|
||||
about = {
|
||||
"website": 'https://duckduckgo.com/',
|
||||
"wikidata_id": 'Q12805',
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# pylint: disable=invalid-name
|
||||
"""Dummy Offline"""
|
||||
|
||||
|
||||
# about
|
||||
about = {
|
||||
"wikidata_id": None,
|
||||
|
||||
@@ -65,7 +65,6 @@ code lines are just relabeled (starting from 1) and appended (a disjoint set of
|
||||
code blocks in a single file might be returned from the API).
|
||||
"""
|
||||
|
||||
|
||||
import typing as t
|
||||
from urllib.parse import urlencode
|
||||
|
||||
|
||||
@@ -9,12 +9,15 @@ engines:
|
||||
- :ref:`google scholar engine`
|
||||
- :ref:`google autocomplete`
|
||||
|
||||
This implementation uses Nokia user agents to request an XML layout from Google.
|
||||
The normal web version requires executing JavaScript to load the results and
|
||||
therefore is currently not used here. See `Google discussion`_ for more
|
||||
information on that topic.
|
||||
|
||||
.. _Google discussion: https://github.com/searxng/searxng/issues/6359
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import time
|
||||
import typing as t
|
||||
from urllib.parse import unquote, urlencode
|
||||
|
||||
@@ -44,16 +47,16 @@ about = {
|
||||
"official_api_documentation": "https://developers.google.com/custom-search/",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
"results": "XML",
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ["general", "web"]
|
||||
paging = True
|
||||
max_page = 50
|
||||
"""`Google max 50 pages`_
|
||||
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
|
||||
|
||||
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
|
||||
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
@@ -64,38 +67,23 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
|
||||
# Filter results. 0: None, 1: Moderate, 2: Strict
|
||||
filter_mapping = {0: "off", 1: "medium", 2: "high"}
|
||||
|
||||
# https://github.com/searxng/searxng/issues/6359
|
||||
nokia_useragents = (
|
||||
"Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
|
||||
"Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
|
||||
"Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
|
||||
"Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
|
||||
"Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
|
||||
"NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
|
||||
)
|
||||
|
||||
|
||||
# specific xpath variables
|
||||
# ------------------------
|
||||
|
||||
# Suggestions are links placed in a *card-section*, we extract only the text
|
||||
# from the links not the links itself.
|
||||
suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
|
||||
|
||||
|
||||
_arcid_range = string.ascii_letters + string.digits + "_-"
|
||||
_arcid_random: tuple[str, int] | None = None
|
||||
|
||||
|
||||
def ui_async(start: int) -> str:
|
||||
"""Format of the response from UI's async request.
|
||||
|
||||
- ``arc_id:<...>,use_ac:true,_fmt:prog``
|
||||
|
||||
The arc_id is random generated every hour.
|
||||
"""
|
||||
global _arcid_random # pylint: disable=global-statement
|
||||
|
||||
use_ac = "use_ac:true"
|
||||
# _fmt:html returns a HTTP 500 when user search for celebrities like
|
||||
# '!google natasha allegri' or '!google chris evans'
|
||||
_fmt = "_fmt:prog"
|
||||
|
||||
# create a new random arc_id every hour
|
||||
if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
|
||||
_arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
|
||||
arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
|
||||
|
||||
return ",".join([arc_id, use_ac, _fmt])
|
||||
suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
|
||||
|
||||
|
||||
def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
|
||||
@@ -127,19 +115,11 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
A instance of :py:obj:`babel.core.Locale` build from the
|
||||
``searxng_locale`` value.
|
||||
|
||||
subdomain:
|
||||
Google subdomain :py:obj:`google_domains` that fits to the country
|
||||
code.
|
||||
|
||||
params:
|
||||
Py-Dictionary with additional request arguments (can be passed to
|
||||
:py:func:`urllib.parse.urlencode`).
|
||||
|
||||
- ``hl`` parameter: specifies the interface language of user interface.
|
||||
- ``lr`` parameter: restricts search results to documents written in
|
||||
a particular language.
|
||||
- ``cr`` parameter: restricts search results to documents
|
||||
originating in a particular country.
|
||||
- ``ie`` parameter: sets the character encoding scheme that should
|
||||
be used to interpret the query string ('utf8').
|
||||
- ``oe`` parameter: sets the character encoding scheme that should
|
||||
@@ -156,7 +136,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
ret_val: dict[str, t.Any] = {
|
||||
"language": None,
|
||||
"country": None,
|
||||
"subdomain": None,
|
||||
"params": {},
|
||||
"headers": {},
|
||||
"cookies": {},
|
||||
@@ -169,7 +148,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
except babel.core.UnknownLocaleError:
|
||||
locale = None
|
||||
|
||||
eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
|
||||
eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
|
||||
lang_code = eng_lang.split("_")[-1] # lang_zh-TW --> zh-TW / lang_en --> en
|
||||
country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
|
||||
|
||||
@@ -184,7 +163,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
ret_val["language"] = eng_lang
|
||||
ret_val["country"] = country
|
||||
ret_val["locale"] = locale
|
||||
ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
|
||||
|
||||
# hl parameter:
|
||||
# The hl parameter specifies the interface language (host language) of
|
||||
@@ -223,9 +201,11 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
|
||||
# specify a region (country) only if a region is given in the selected
|
||||
# locale --> https://github.com/searxng/searxng/issues/2672
|
||||
ret_val["params"]["cr"] = ""
|
||||
if len(sxng_locale.split("-")) > 1:
|
||||
ret_val["params"]["cr"] = "country" + country
|
||||
|
||||
if country is not None:
|
||||
ret_val["params"]["cr"] = ""
|
||||
if len(sxng_locale.split("-")) > 1:
|
||||
ret_val["params"]["cr"] = "country" + country
|
||||
|
||||
# gl parameter: (mandatory by Google News)
|
||||
# The gl parameter value is a two-letter country code. For WebSearch
|
||||
@@ -300,88 +280,77 @@ def detect_google_sorry(resp: "SXNG_Response"):
|
||||
raise SearxEngineCaptchaException()
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
"""Google search request"""
|
||||
# pylint: disable=line-too-long
|
||||
start = (params["pageno"] - 1) * 10
|
||||
google_info = get_google_info(params, traits)
|
||||
|
||||
# https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
|
||||
query_url = (
|
||||
"https://"
|
||||
+ google_info["subdomain"]
|
||||
+ "/search"
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
"q": query,
|
||||
**google_info["params"],
|
||||
"filter": "0",
|
||||
"start": start,
|
||||
# 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
|
||||
# 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
|
||||
# 'cs' : 1,
|
||||
# 'sa': 'N',
|
||||
# 'yv': 3,
|
||||
# 'prmd': 'vin',
|
||||
# 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
|
||||
# 'sa': 'N',
|
||||
# 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
|
||||
# formally known as use_mobile_ui
|
||||
# "asearch": "arc",
|
||||
# "async": str_async,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if params["time_range"] in time_range_dict:
|
||||
query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
|
||||
if params["safesearch"]:
|
||||
query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
|
||||
params["url"] = query_url
|
||||
|
||||
params["cookies"] = google_info["cookies"]
|
||||
params["headers"].update(google_info["headers"])
|
||||
def unwrap_google_url(raw_url: str) -> str:
|
||||
# remove redirector from url
|
||||
if raw_url.startswith("/url?q="):
|
||||
return unquote(raw_url[7:].split("&sa=U")[0])
|
||||
return raw_url
|
||||
|
||||
|
||||
# regex match to get image map that is found inside the returned javascript:
|
||||
# (function(){var s='...';var i=['...'] ...}
|
||||
RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
|
||||
|
||||
|
||||
def parse_url_images(text: str):
|
||||
data_image_map = {}
|
||||
|
||||
for image_url, img_id in RE_DATA_IMAGE.findall(text):
|
||||
data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
|
||||
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
|
||||
return data_image_map
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
"""Get response from google's search request"""
|
||||
# pylint: disable=too-many-branches, too-many-statements
|
||||
def wml_dom(resp: "SXNG_Response"):
|
||||
detect_google_sorry(resp)
|
||||
data_image_map = parse_url_images(resp.text)
|
||||
text = resp.text
|
||||
if text.lstrip().startswith("<?xml"):
|
||||
text = text.split("?>", 1)[-1]
|
||||
return html.fromstring(text)
|
||||
|
||||
|
||||
def google_request(
|
||||
query: str,
|
||||
params: "OnlineParams",
|
||||
extra_args: dict[str, t.Any] | None = None,
|
||||
*,
|
||||
eng_traits: EngineTraits | None = None,
|
||||
use_time_range: bool = True,
|
||||
use_safesearch: bool = True,
|
||||
safesearch_map: dict[int, str] | None = None,
|
||||
use_locales: bool = True,
|
||||
) -> None:
|
||||
google_info = get_google_info(params, eng_traits or traits)
|
||||
if not use_locales:
|
||||
google_info["params"].pop("lr")
|
||||
google_info["params"].pop("cr")
|
||||
|
||||
start = (params["pageno"] - 1) * 10
|
||||
args: dict[str, t.Any] = {
|
||||
"q": query,
|
||||
"sca_esv": "1",
|
||||
**google_info["params"],
|
||||
**(extra_args or {}),
|
||||
}
|
||||
if start:
|
||||
args["start"] = start
|
||||
if use_time_range and params["time_range"] in time_range_dict:
|
||||
args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
|
||||
if use_safesearch and params["safesearch"]:
|
||||
args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
|
||||
|
||||
params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
|
||||
params["headers"]["User-Agent"] = random.choice(nokia_useragents)
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
google_request(query, params)
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
results = EngineResults()
|
||||
|
||||
# convert the text to dom
|
||||
dom = html.fromstring(resp.text)
|
||||
dom = wml_dom(resp)
|
||||
|
||||
# parse results
|
||||
for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
|
||||
# pylint: disable=too-many-nested-blocks
|
||||
for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
|
||||
|
||||
try:
|
||||
title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
|
||||
title_tag = eval_xpath_getindex(
|
||||
result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
|
||||
)
|
||||
if title_tag is None:
|
||||
# this not one of the common google results *section*
|
||||
logger.debug("ignoring item from the result_xpath list: missing title")
|
||||
continue
|
||||
title = extract_text(title_tag)
|
||||
|
||||
raw_url = result.get("href")
|
||||
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
|
||||
if raw_url is None:
|
||||
logger.debug(
|
||||
'ignoring item from the result_xpath list: missing url of title "%s"',
|
||||
@@ -389,30 +358,19 @@ def response(resp: "SXNG_Response"):
|
||||
)
|
||||
continue
|
||||
|
||||
if raw_url.startswith('/url?q='):
|
||||
url = unquote(raw_url[7:].split("&sa=U")[0]) # remove the google redirector
|
||||
else:
|
||||
url = raw_url
|
||||
|
||||
content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
|
||||
for item in content_nodes:
|
||||
for script in item.xpath(".//script"):
|
||||
script.getparent().remove(script)
|
||||
|
||||
content = extract_text(content_nodes[0])
|
||||
|
||||
# Images that are NOT the favicon
|
||||
xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
|
||||
|
||||
thumbnail = None
|
||||
if xpath_image is not None:
|
||||
thumbnail = xpath_image.get("src")
|
||||
if thumbnail.startswith("data:image"):
|
||||
img_id = xpath_image.get("id")
|
||||
if img_id:
|
||||
thumbnail = data_image_map.get(img_id)
|
||||
|
||||
results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
|
||||
url = unwrap_google_url(raw_url)
|
||||
content = extract_text(
|
||||
eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
|
||||
)
|
||||
thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
url=url,
|
||||
title=title or "",
|
||||
content=content or "",
|
||||
thumbnail=thumbnail or "",
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
logger.error(e, exc_info=True)
|
||||
@@ -420,10 +378,8 @@ def response(resp: "SXNG_Response"):
|
||||
|
||||
# parse suggestion
|
||||
for suggestion in eval_xpath_list(dom, suggestion_xpath):
|
||||
# append suggestion
|
||||
results.append({"suggestion": extract_text(suggestion)})
|
||||
results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
|
||||
|
||||
# return results
|
||||
return results
|
||||
|
||||
|
||||
@@ -456,14 +412,12 @@ skip_countries = [
|
||||
]
|
||||
|
||||
|
||||
def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
|
||||
def fetch_traits(engine_traits: EngineTraits):
|
||||
"""Fetch languages from Google."""
|
||||
# pylint: disable=import-outside-toplevel, too-many-branches
|
||||
|
||||
from searx.network import get # see https://github.com/searxng/searxng/issues/762
|
||||
|
||||
engine_traits.custom["supported_domains"] = {}
|
||||
|
||||
resp = get("https://www.google.com/preferences", timeout=5)
|
||||
if not resp.ok:
|
||||
raise RuntimeError("Response from Google preferences is not OK.")
|
||||
@@ -514,22 +468,3 @@ def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
|
||||
|
||||
# alias regions
|
||||
engine_traits.regions["zh-CN"] = "HK"
|
||||
|
||||
# supported domains
|
||||
|
||||
if add_domains:
|
||||
resp = get("https://www.google.com/supported_domains", timeout=5)
|
||||
if not resp.ok:
|
||||
raise RuntimeError("Response from Google supported domains is not OK.")
|
||||
|
||||
for domain in resp.text.split():
|
||||
domain = domain.strip()
|
||||
if not domain or domain in [
|
||||
".google.com",
|
||||
]:
|
||||
continue
|
||||
region = domain.split(".")[-1].upper()
|
||||
engine_traits.custom["supported_domains"][region] = "www" + domain
|
||||
if region == "HK":
|
||||
# There is no google.cn, we use .com.hk for zh-CN
|
||||
engine_traits.custom["supported_domains"]["CN"] = "www" + domain
|
||||
|
||||
@@ -95,12 +95,11 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
token = _cse_token()
|
||||
|
||||
google_info = get_google_info(params, traits)
|
||||
info: dict[str, str] = google_info["params"]
|
||||
|
||||
args = {
|
||||
"rsz": "filtered_cse",
|
||||
"num": str(page_size),
|
||||
"hl": info["hl"],
|
||||
"hl": google_info["params"]["hl"],
|
||||
"cselibv": token["cselibv"],
|
||||
"cx": CX,
|
||||
"q": query,
|
||||
@@ -114,10 +113,6 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
|
||||
args["sort"] = f"date:r:{start_date}:{end_date}"
|
||||
|
||||
if info.get("lr"):
|
||||
args["lr"] = info["lr"]
|
||||
if info.get("cr"):
|
||||
args["cr"] = info["cr"]
|
||||
if google_info["country"] not in (None, "ZZ"):
|
||||
args["gl"] = google_info["country"]
|
||||
if token["exp"]:
|
||||
|
||||
@@ -1,122 +1,75 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""This is the implementation of the Google Images engine using the internal
|
||||
Google API used by the Google Go Android app.
|
||||
"""Google Images: see :py:obj:`searx.engines.google`."""
|
||||
|
||||
This internal API offer results in
|
||||
|
||||
- JSON (``_fmt:json``)
|
||||
- Protobuf_ (``_fmt:pb``)
|
||||
- Protobuf_ compressed? (``_fmt:pc``)
|
||||
- HTML (``_fmt:html``)
|
||||
- Protobuf_ encoded in JSON (``_fmt:jspb``).
|
||||
|
||||
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode
|
||||
from json import loads
|
||||
import typing as t
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import (
|
||||
get_google_info,
|
||||
time_range_dict,
|
||||
detect_google_sorry,
|
||||
)
|
||||
from searx.engines.google import google_request, wml_dom
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import eval_xpath_list
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": 'https://images.google.com',
|
||||
"wikidata_id": 'Q521550',
|
||||
"official_api_documentation": 'https://developers.google.com/custom-search',
|
||||
"website": "https://images.google.com",
|
||||
"wikidata_id": "Q521550",
|
||||
"official_api_documentation": "https://developers.google.com/custom-search",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'JSON',
|
||||
"results": "XML",
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ['images', 'web']
|
||||
categories = ["images", "web"]
|
||||
paging = True
|
||||
max_page = 50
|
||||
"""`Google max 50 pages`_
|
||||
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
|
||||
|
||||
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
|
||||
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
safesearch = True
|
||||
|
||||
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
|
||||
filter_mapping = {0: "images", 1: "active", 2: "active"}
|
||||
|
||||
|
||||
def request(query, params):
|
||||
"""Google-Image search request"""
|
||||
|
||||
google_info = get_google_info(params, traits)
|
||||
|
||||
query_url = (
|
||||
'https://'
|
||||
+ google_info['subdomain']
|
||||
+ '/search'
|
||||
+ '?'
|
||||
+ urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
|
||||
# don't urlencode this because wildly different AND bad results
|
||||
# pagination uses Zero-based numbering
|
||||
+ f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
google_request(
|
||||
query,
|
||||
params,
|
||||
{"tbm": "isch"},
|
||||
eng_traits=traits,
|
||||
safesearch_map=filter_mapping,
|
||||
use_locales=False,
|
||||
)
|
||||
|
||||
if params['time_range'] in time_range_dict:
|
||||
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
|
||||
if params['safesearch']:
|
||||
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
|
||||
params['url'] = query_url
|
||||
params['cookies'] = google_info['cookies']
|
||||
params['headers'].update(google_info['headers'])
|
||||
# this ua will allow getting ~50 results instead of 10. #1641
|
||||
params['headers']['User-Agent'] = (
|
||||
'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
|
||||
)
|
||||
|
||||
return params
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
results = EngineResults()
|
||||
dom = wml_dom(resp)
|
||||
|
||||
|
||||
def response(resp):
|
||||
"""Get response from google's search request"""
|
||||
results = []
|
||||
|
||||
detect_google_sorry(resp)
|
||||
|
||||
json_start = resp.text.find('{"ischj":')
|
||||
json_data = loads(resp.text[json_start:])
|
||||
|
||||
for item in json_data["ischj"].get("metadata", []):
|
||||
result_item = {
|
||||
'url': item["result"]["referrer_url"],
|
||||
'title': item["result"]["page_title"],
|
||||
'content': item["text_in_grid"]["snippet"],
|
||||
'source': item["result"]["site_title"],
|
||||
'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
|
||||
'img_src': item["original_image"]["url"],
|
||||
'thumbnail_src': item["thumbnail"]["url"],
|
||||
'template': 'images.html',
|
||||
}
|
||||
|
||||
author = item["result"].get('iptc', {}).get('creator')
|
||||
if author:
|
||||
result_item['author'] = ', '.join(author)
|
||||
|
||||
copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
|
||||
if copyright_notice:
|
||||
result_item['source'] += ' | ' + copyright_notice
|
||||
|
||||
freshness_date = item["result"].get("freshness_date")
|
||||
if freshness_date:
|
||||
result_item['source'] += ' | ' + freshness_date
|
||||
|
||||
file_size = item.get('gsa', {}).get('file_size')
|
||||
if file_size:
|
||||
result_item['source'] += ' (%s)' % file_size
|
||||
|
||||
results.append(result_item)
|
||||
for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
|
||||
qs = parse_qs(urlparse(link.get("href", "")).query)
|
||||
img_src = qs.get("imgurl", [""])[0]
|
||||
url = qs.get("imgrefurl", [""])[0]
|
||||
if not img_src or not url:
|
||||
continue
|
||||
width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
|
||||
tbnid = qs.get("tbnid", [""])[0]
|
||||
results.add(
|
||||
results.types.Image(
|
||||
url=url,
|
||||
title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
|
||||
img_src=img_src,
|
||||
thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
|
||||
resolution=f"{width} x {height}" if width and height else "",
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
@@ -1,324 +1,91 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""This is the implementation of the Google News engine.
|
||||
"""Google News: see :py:obj:`searx.engines.google`."""
|
||||
|
||||
Google News has a different region handling compared to Google WEB.
|
||||
|
||||
- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
|
||||
- the hl_ argument has to be set correctly (and different to Google WEB)
|
||||
- the gl_ argument is mandatory
|
||||
|
||||
If one of this argument is not set correctly, the request is redirected to
|
||||
CONSENT dialog::
|
||||
|
||||
https://consent.google.com/m?continue=
|
||||
|
||||
The google news API ignores some parameters from the common :ref:`google API`:
|
||||
|
||||
- num_ : the number of search results is ignored / there is no paging all
|
||||
results for a query term are in the first response.
|
||||
- save_ : is ignored / Google-News results are always *SafeSearch*
|
||||
|
||||
.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
|
||||
.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
|
||||
.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
|
||||
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
|
||||
"""
|
||||
import typing as t
|
||||
|
||||
import json
|
||||
import base64
|
||||
from urllib.parse import urlencode
|
||||
from lxml import html
|
||||
import babel
|
||||
|
||||
from searx import locales
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import google_request, unwrap_google_url, wml_dom
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import (
|
||||
eval_xpath,
|
||||
eval_xpath_list,
|
||||
eval_xpath_getindex,
|
||||
eval_xpath_list,
|
||||
extract_text,
|
||||
)
|
||||
|
||||
from searx.engines.google import fetch_traits as _fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import (
|
||||
get_google_info,
|
||||
detect_google_sorry,
|
||||
)
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": "https://news.google.com",
|
||||
"website": "https://www.google.com",
|
||||
"wikidata_id": "Q12020",
|
||||
"official_api_documentation": "https://developers.google.com/custom-search",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
"results": "XML",
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ["news"]
|
||||
paging = False
|
||||
paging = True
|
||||
max_page = 50
|
||||
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
|
||||
|
||||
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
time_range_support = False
|
||||
language_support = True
|
||||
|
||||
# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
|
||||
# False here.
|
||||
#
|
||||
# safesearch : results are identical for safesearch=0 and safesearch=2
|
||||
safesearch = True
|
||||
base_url: str = "https://news.google.com"
|
||||
safesearch = False
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
"""Google-News search request"""
|
||||
|
||||
sxng_locale = params.get("searxng_locale", "en-US")
|
||||
ceid: str = locales.get_engine_locale(
|
||||
sxng_locale, traits.custom["ceid"], default="US:en"
|
||||
) # pyright: ignore[reportAssignmentType]
|
||||
google_info = get_google_info(params, traits)
|
||||
google_info["subdomain"] = "news.google.com" # google news has only one domain
|
||||
|
||||
ceid_region, ceid_lang = ceid.split(":")
|
||||
ceid_lang, ceid_suffix = (
|
||||
ceid_lang.split(":")
|
||||
+ [
|
||||
"",
|
||||
]
|
||||
)[:2]
|
||||
|
||||
google_info["params"]["hl"] = ceid_lang
|
||||
|
||||
if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
|
||||
|
||||
if ceid_region.lower() == ceid_lang:
|
||||
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
|
||||
else:
|
||||
google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
|
||||
|
||||
elif ceid_region.lower() != ceid_lang:
|
||||
|
||||
if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
|
||||
google_info["params"]["hl"] = ceid_lang
|
||||
else:
|
||||
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
|
||||
|
||||
google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
|
||||
google_info["params"]["gl"] = ceid_region
|
||||
|
||||
query_url = (
|
||||
"https://"
|
||||
+ google_info["subdomain"]
|
||||
+ "/search?"
|
||||
+ urlencode(
|
||||
{"q": query, **google_info["params"]},
|
||||
)
|
||||
# ceid includes a ':' character which must not be urlencoded
|
||||
+ ("&ceid=%s" % ceid)
|
||||
google_request(
|
||||
query,
|
||||
params,
|
||||
{"tbm": "nws"},
|
||||
eng_traits=traits,
|
||||
use_time_range=False,
|
||||
use_safesearch=False,
|
||||
use_locales=False,
|
||||
)
|
||||
|
||||
params["url"] = query_url
|
||||
params["cookies"] = google_info["cookies"]
|
||||
params["headers"].update(google_info["headers"])
|
||||
|
||||
def _span_text(link, css_class: str):
|
||||
return extract_text(
|
||||
eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
|
||||
allow_none=True,
|
||||
)
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
"""Get response from google's search request"""
|
||||
|
||||
res = EngineResults()
|
||||
|
||||
detect_google_sorry(resp)
|
||||
|
||||
# convert the text to dom
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
|
||||
|
||||
url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
|
||||
if not url:
|
||||
continue
|
||||
if url.startswith("./"):
|
||||
url = base_url + url[1:]
|
||||
|
||||
# The real URL is often encoded in the "jslog" attribute
|
||||
jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
|
||||
|
||||
# Try to extract the real URL from jslog
|
||||
real_url: str | None = None
|
||||
if jslog:
|
||||
# jslog format is usually: "95014; 5:<base64>; track:click,vis". We
|
||||
# want the second part (index 1) after splitting by ";"
|
||||
parts: list[str] = jslog.split(";")
|
||||
if len(parts) > 1:
|
||||
b64_data: str = parts[1].split(":")[-1].strip()
|
||||
# Pad base64 if necessary
|
||||
b64_data += "=" * (-len(b64_data) % 4)
|
||||
decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
|
||||
# The URL is typically the last element in the decoded array
|
||||
if (
|
||||
isinstance(decoded_data, list)
|
||||
and isinstance(decoded_data[-1], str)
|
||||
and decoded_data[-1].startswith("http")
|
||||
):
|
||||
real_url = decoded_data[-1]
|
||||
if real_url:
|
||||
url = real_url
|
||||
else:
|
||||
logger.error(f"no real-url found: {url}")
|
||||
results = EngineResults()
|
||||
seen = set()
|
||||
for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
|
||||
href = link.get("href")
|
||||
if not href:
|
||||
continue
|
||||
|
||||
title = extract_text(eval_xpath(result, "./h4")) or ""
|
||||
url = unwrap_google_url(href)
|
||||
if url in seen or "google.com/search" in url:
|
||||
continue
|
||||
|
||||
# The pub_date is mostly a string like 'yesterday', not a real timezone
|
||||
# date or time. Therefore we can't use publishedDate and place the
|
||||
# *pub* sting into the content.
|
||||
title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
|
||||
if not title:
|
||||
continue
|
||||
|
||||
pub_date = extract_text(eval_xpath(result, ".//time"))
|
||||
pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
|
||||
content = " / ".join([x for x in [pub_origin, pub_date] if x])
|
||||
source = _span_text(link, "dXDvrc")
|
||||
pub_date = _span_text(link, "YVIcad")
|
||||
thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
|
||||
|
||||
thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
|
||||
if thumbnail and thumbnail.startswith("/"):
|
||||
thumbnail = base_url + thumbnail
|
||||
|
||||
res.add(
|
||||
res.types.MainResult(
|
||||
seen.add(url)
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
url=url,
|
||||
title=title,
|
||||
content=content,
|
||||
thumbnail=thumbnail,
|
||||
content=" / ".join(x for x in [source, pub_date] if x),
|
||||
thumbnail=thumbnail or "",
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
ceid_list = [
|
||||
"AE:ar",
|
||||
"AR:es-419",
|
||||
"AT:de",
|
||||
"AU:en",
|
||||
"BD:bn",
|
||||
"BE:fr",
|
||||
"BE:nl",
|
||||
"BG:bg",
|
||||
"BR:pt-419",
|
||||
"BW:en",
|
||||
"CA:en",
|
||||
"CA:fr",
|
||||
"CH:de",
|
||||
"CH:fr",
|
||||
"CL:es-419",
|
||||
"CN:zh-Hans",
|
||||
"CO:es-419",
|
||||
"CU:es-419",
|
||||
"CZ:cs",
|
||||
"DE:de",
|
||||
"EE:et",
|
||||
"EG:ar",
|
||||
"ES:ca",
|
||||
"ES:es",
|
||||
"ET:en",
|
||||
"FI:fi",
|
||||
"FR:fr",
|
||||
"GB:en",
|
||||
"GH:en",
|
||||
"GR:el",
|
||||
"HK:zh-Hant",
|
||||
"HU:hu",
|
||||
"ID:en",
|
||||
"ID:id",
|
||||
"IE:en",
|
||||
"IL:en",
|
||||
"IL:he",
|
||||
"IN:bn",
|
||||
"IN:en",
|
||||
"IN:gu",
|
||||
"IN:hi",
|
||||
"IN:ml",
|
||||
"IN:mr",
|
||||
"IN:pa",
|
||||
"IN:ta",
|
||||
"IN:te",
|
||||
"IT:it",
|
||||
"JP:ja",
|
||||
"KE:en",
|
||||
"KR:ko",
|
||||
"LB:ar",
|
||||
"LT:lt",
|
||||
"LV:en",
|
||||
"LV:lv",
|
||||
"MA:fr",
|
||||
"MY:en",
|
||||
"MY:ms",
|
||||
"NA:en",
|
||||
"NG:en",
|
||||
"NL:nl",
|
||||
"NO:no",
|
||||
"NZ:en",
|
||||
"PH:en",
|
||||
"PK:en",
|
||||
"PL:pl",
|
||||
"RO:ro",
|
||||
"RS:sr",
|
||||
"RU:ru",
|
||||
"SA:ar",
|
||||
"SE:sv",
|
||||
"SG:en",
|
||||
"SI:sl",
|
||||
"SK:sk",
|
||||
"SN:fr",
|
||||
"TH:th",
|
||||
"TR:tr",
|
||||
"TZ:en",
|
||||
"UA:ru",
|
||||
"UA:uk",
|
||||
"UG:en",
|
||||
"US:en",
|
||||
"VN:vi",
|
||||
"ZA:en",
|
||||
"ZW:en",
|
||||
]
|
||||
"""List of region/language combinations supported by Google News. Values of the
|
||||
``ceid`` argument of the Google News REST API."""
|
||||
|
||||
|
||||
_skip_values = [
|
||||
"ET:en", # english (ethiopia)
|
||||
"ID:en", # english (indonesia)
|
||||
"LV:en", # english (latvia)
|
||||
]
|
||||
|
||||
_ceid_locale_map = {"NO:no": "nb-NO"}
|
||||
|
||||
|
||||
def fetch_traits(engine_traits: EngineTraits):
|
||||
_fetch_traits(engine_traits, add_domains=False)
|
||||
|
||||
engine_traits.custom["ceid"] = {}
|
||||
|
||||
for ceid in ceid_list:
|
||||
if ceid in _skip_values:
|
||||
continue
|
||||
|
||||
region, lang = ceid.split(":")
|
||||
x = lang.split("-")
|
||||
if len(x) > 1:
|
||||
if x[1] not in ["Hant", "Hans"]:
|
||||
lang = x[0]
|
||||
|
||||
sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
|
||||
try:
|
||||
locale = babel.Locale.parse(sxng_locale, sep="-")
|
||||
except babel.UnknownLocaleError:
|
||||
print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
|
||||
continue
|
||||
|
||||
engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid
|
||||
return results
|
||||
|
||||
@@ -77,8 +77,6 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
"""Google-Scholar search request"""
|
||||
|
||||
google_info = get_google_info(params, traits)
|
||||
# subdomain is: scholar.google.xy
|
||||
google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
|
||||
|
||||
args = {
|
||||
"q": query,
|
||||
@@ -89,7 +87,7 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
}
|
||||
args.update(time_range_args(params))
|
||||
|
||||
params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
|
||||
params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
|
||||
params["cookies"] = google_info["cookies"]
|
||||
params["headers"].update(google_info["headers"])
|
||||
|
||||
|
||||
@@ -1,185 +1,87 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""This is the implementation of the Google Videos engine.
|
||||
"""Google Videos: see :py:obj:`searx.engines.google`."""
|
||||
|
||||
.. admonition:: Content-Security-Policy (CSP)
|
||||
|
||||
This engine needs to allow images from the `data URLs`_ (prefixed with the
|
||||
``data:`` scheme)::
|
||||
|
||||
Header set Content-Security-Policy "img-src 'self' data: ;"
|
||||
|
||||
.. _data URLs:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
|
||||
"""
|
||||
import re
|
||||
from urllib.parse import urlencode, urlparse, parse_qs, unquote
|
||||
from lxml import html
|
||||
|
||||
from searx.utils import (
|
||||
eval_xpath_list,
|
||||
eval_xpath_getindex,
|
||||
extract_text,
|
||||
)
|
||||
import typing as t
|
||||
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import (
|
||||
get_google_info,
|
||||
time_range_dict,
|
||||
filter_mapping,
|
||||
suggestion_xpath,
|
||||
detect_google_sorry,
|
||||
ui_async,
|
||||
from searx.engines.google import google_request, unwrap_google_url, wml_dom
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import (
|
||||
eval_xpath_getindex,
|
||||
eval_xpath_list,
|
||||
extract_text,
|
||||
get_embeded_stream_url,
|
||||
parse_duration_string,
|
||||
)
|
||||
from searx.utils import get_embeded_stream_url
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": 'https://www.google.com',
|
||||
"wikidata_id": 'Q219885',
|
||||
"official_api_documentation": 'https://developers.google.com/custom-search',
|
||||
"website": "https://www.google.com",
|
||||
"wikidata_id": "Q219885",
|
||||
"official_api_documentation": "https://developers.google.com/custom-search",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'HTML',
|
||||
"results": "XML",
|
||||
}
|
||||
|
||||
# engine dependent config
|
||||
categories = ['videos', 'web']
|
||||
categories = ["videos", "web"]
|
||||
paging = True
|
||||
max_page = 50
|
||||
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
|
||||
|
||||
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
|
||||
"""
|
||||
language_support = True
|
||||
time_range_support = True
|
||||
safesearch = True
|
||||
|
||||
|
||||
# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
|
||||
# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
|
||||
RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
|
||||
|
||||
|
||||
def parse_data_images(text: str):
|
||||
data_image_map = {}
|
||||
|
||||
for img_id, data_image in RE_DATA_IMAGE.findall(text):
|
||||
end_pos = data_image.rfind("=")
|
||||
if end_pos > 0:
|
||||
data_image = data_image[: end_pos + 1]
|
||||
data_image_map[img_id] = data_image
|
||||
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
|
||||
return data_image_map
|
||||
|
||||
|
||||
def request(query, params):
|
||||
"""Google-Video search request"""
|
||||
google_info = get_google_info(params, traits)
|
||||
start = (params['pageno'] - 1) * 10
|
||||
|
||||
query_url = (
|
||||
'https://'
|
||||
+ google_info['subdomain']
|
||||
+ '/search'
|
||||
+ "?"
|
||||
+ urlencode(
|
||||
{
|
||||
'q': query,
|
||||
'tbm': "vid",
|
||||
'start': start,
|
||||
**google_info['params'],
|
||||
'asearch': 'arc',
|
||||
'async': ui_async(start),
|
||||
}
|
||||
)
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
google_request(
|
||||
query,
|
||||
params,
|
||||
{"tbm": "vid"},
|
||||
eng_traits=traits,
|
||||
use_locales=False,
|
||||
)
|
||||
|
||||
if params['time_range'] in time_range_dict:
|
||||
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
|
||||
if 'safesearch' in params:
|
||||
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
|
||||
params['url'] = query_url
|
||||
|
||||
params['cookies'] = google_info['cookies']
|
||||
params['headers'].update(google_info['headers'])
|
||||
return params
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
results = EngineResults()
|
||||
|
||||
|
||||
def response(resp):
|
||||
"""Get response from google's search request"""
|
||||
results = []
|
||||
|
||||
detect_google_sorry(resp)
|
||||
data_image_map = parse_data_images(resp.text)
|
||||
|
||||
# convert the text to dom
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
|
||||
|
||||
# parse results
|
||||
for result in result_divs:
|
||||
for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
|
||||
title = extract_text(
|
||||
eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
|
||||
eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
|
||||
allow_none=True,
|
||||
)
|
||||
url = eval_xpath_getindex(
|
||||
result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
|
||||
)
|
||||
if url and url.startswith('/url?q='):
|
||||
url = unquote(url[7:].split('&sa=U')[0])
|
||||
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
|
||||
if not title or not raw_url:
|
||||
continue
|
||||
|
||||
content = extract_text(
|
||||
eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
|
||||
)
|
||||
pub_info = extract_text(
|
||||
eval_xpath_getindex(
|
||||
result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
|
||||
),
|
||||
allow_none=True,
|
||||
)
|
||||
# Broader XPath to find any <img> element
|
||||
thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
|
||||
duration = extract_text(
|
||||
eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
|
||||
)
|
||||
video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
|
||||
url = unwrap_google_url(raw_url)
|
||||
thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
|
||||
if "/default.jpg" in thumbnail:
|
||||
thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
|
||||
length = None
|
||||
for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
|
||||
length = parse_duration_string(extract_text(span) or "")
|
||||
if length:
|
||||
break
|
||||
|
||||
# Fallback for video_id from URL if not found via XPath
|
||||
if not video_id and url and 'youtube.com' in url:
|
||||
parsed_url = urlparse(url)
|
||||
video_id = parse_qs(parsed_url.query).get('v', [None])[0]
|
||||
|
||||
# Handle thumbnail
|
||||
if thumbnail and thumbnail.startswith('data:image'):
|
||||
img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
|
||||
if img_id and img_id in data_image_map:
|
||||
thumbnail = data_image_map[img_id]
|
||||
else:
|
||||
thumbnail = None
|
||||
if not thumbnail and video_id:
|
||||
thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
|
||||
|
||||
# Handle video embed URL
|
||||
embed_url = None
|
||||
if video_id:
|
||||
embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
|
||||
elif url:
|
||||
embed_url = get_embeded_stream_url(url)
|
||||
|
||||
# Only append results with valid title and url
|
||||
if title and url:
|
||||
results.append(
|
||||
{
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content': content or '',
|
||||
'author': pub_info,
|
||||
'thumbnail': thumbnail,
|
||||
'length': duration,
|
||||
'iframe_src': embed_url,
|
||||
'template': 'videos.html',
|
||||
}
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
url=url,
|
||||
title=title,
|
||||
thumbnail=thumbnail,
|
||||
length=length,
|
||||
iframe_src=get_embeded_stream_url(url) or "",
|
||||
template="videos.html",
|
||||
)
|
||||
|
||||
# parse suggestion
|
||||
for suggestion in eval_xpath_list(dom, suggestion_xpath):
|
||||
results.append({'suggestion': extract_text(suggestion)})
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
from urllib.parse import urlencode
|
||||
from dateutil import parser
|
||||
|
||||
|
||||
about = {
|
||||
# pylint: disable=line-too-long
|
||||
"website": "https://hex.pm/",
|
||||
|
||||
@@ -108,14 +108,12 @@ def get_infobox(alt_forms, result_url, definitions):
|
||||
infobox_content.append(f'<p><i>Other forms:</i> {", ".join(alt_forms[1:])}</p>')
|
||||
|
||||
# definitions
|
||||
infobox_content.append(
|
||||
'''
|
||||
infobox_content.append('''
|
||||
<small><a href="https://www.edrdg.org/wiki/index.php/JMdict-EDICT_Dictionary_Project">JMdict</a>
|
||||
and <a href="https://www.edrdg.org/enamdict/enamdict_doc.html">JMnedict</a>
|
||||
by <a href="https://www.edrdg.org/edrdg/licence.html">EDRDG</a>, CC BY-SA 3.0.</small>
|
||||
<ul>
|
||||
'''
|
||||
)
|
||||
''')
|
||||
for pos, engdef, extra in definitions:
|
||||
if pos == 'Wikipedia definition':
|
||||
infobox_content.append('</ul><small>Wikipedia, CC BY-SA 3.0.</small><ul>')
|
||||
|
||||
@@ -49,7 +49,6 @@ except ImportError:
|
||||
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
|
||||
engine_type = 'offline'
|
||||
|
||||
# mongodb connection variables
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
from urllib.parse import urlencode
|
||||
from dateutil import parser
|
||||
|
||||
|
||||
about = {
|
||||
"website": "https://npms.io/",
|
||||
"wikidata_id": "Q7067518",
|
||||
|
||||
@@ -9,7 +9,6 @@ from datetime import datetime
|
||||
from searx.result_types import EngineResults, WeatherAnswer
|
||||
from searx import weather
|
||||
|
||||
|
||||
about = {
|
||||
"website": "https://open-meteo.com",
|
||||
"wikidata_id": None,
|
||||
|
||||
@@ -10,7 +10,8 @@ from flask_babel import gettext
|
||||
|
||||
from searx.data import OSM_KEYS_TAGS, CURRENCIES
|
||||
from searx.external_urls import get_external_url
|
||||
from searx.engines.wikidata import send_wikidata_query, sparql_string_escape, get_thumbnail
|
||||
from searx.wikidata import send_wikidata_query
|
||||
from searx.engines.wikidata import sparql_string_escape, get_thumbnail
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
# about
|
||||
@@ -290,7 +291,8 @@ def get_title_address(result):
|
||||
'house_number': address_raw.get('house_number'),
|
||||
'road': address_raw.get('road'),
|
||||
'locality': address_raw.get(
|
||||
'city', address_raw.get('town', address_raw.get('village')) # noqa
|
||||
'city',
|
||||
address_raw.get('town', address_raw.get('village')), # noqa
|
||||
), # noqa
|
||||
'postcode': address_raw.get('postcode'),
|
||||
'country': address_raw.get('country'),
|
||||
|
||||
@@ -8,7 +8,6 @@ Openverse (formerly known as: Creative Commons search engine) [Images]
|
||||
from json import loads
|
||||
from urllib.parse import urlencode
|
||||
|
||||
|
||||
about = {
|
||||
"website": 'https://openverse.org/',
|
||||
"wikidata_id": None,
|
||||
|
||||
@@ -12,7 +12,6 @@ from searx.enginelib import EngineCache
|
||||
from searx.exceptions import SearxEngineAPIException, SearxEngineAccessDeniedException
|
||||
from searx.network import get
|
||||
|
||||
|
||||
# about
|
||||
about = {
|
||||
"website": 'https://www.pexels.com',
|
||||
|
||||
@@ -48,7 +48,6 @@ Implementations
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import time
|
||||
import random
|
||||
from urllib.parse import urlencode
|
||||
|
||||
@@ -18,7 +18,6 @@ from searx.utils import eval_xpath_list, eval_xpath, extract_text, get_embeded_s
|
||||
from searx.locales import region_tag
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from lxml.etree import ElementBase
|
||||
from searx.extended_types import SXNG_Response
|
||||
|
||||
@@ -35,6 +35,7 @@ Implementations
|
||||
===============
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from datetime import date, timedelta
|
||||
|
||||
@@ -34,7 +34,6 @@ from searx.exceptions import SearxEngineAPIException
|
||||
from searx.result_types import EngineResults
|
||||
from searx.extended_types import SXNG_Response
|
||||
|
||||
|
||||
base_url = 'http://localhost:8983'
|
||||
collection = ''
|
||||
rows = 10
|
||||
|
||||
@@ -117,7 +117,7 @@ def response(resp):
|
||||
|
||||
def init(engine_settings): # pylint: disable=unused-argument
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"]) # type:ignore
|
||||
CACHE = EngineCache(engine_settings["name"]) # type: ignore
|
||||
|
||||
|
||||
def get_client_id() -> str | None:
|
||||
|
||||
@@ -44,6 +44,7 @@ Implementations
|
||||
===============
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
import sqlite3
|
||||
import contextlib
|
||||
|
||||
@@ -82,6 +82,7 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
|
||||
Supported categories are ``web``, ``news`` and ``images``.
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=too-many-statements
|
||||
|
||||
import re
|
||||
|
||||
@@ -74,7 +74,6 @@ Implementations
|
||||
===============
|
||||
"""
|
||||
|
||||
|
||||
from urllib.parse import urlencode
|
||||
from dateutil.parser import parse
|
||||
from searx.utils import html_to_text, humanize_number
|
||||
|
||||
@@ -12,7 +12,6 @@ from lxml import html
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import eval_xpath_list, eval_xpath, extract_text
|
||||
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from lxml.etree import ElementBase
|
||||
from searx.extended_types import SXNG_Response
|
||||
|
||||
@@ -3,28 +3,34 @@
|
||||
|
||||
Some implementations are shared from :ref:`wikipedia engine`.
|
||||
"""
|
||||
|
||||
# pylint: disable=missing-class-docstring
|
||||
|
||||
import typing as t
|
||||
|
||||
import os
|
||||
from hashlib import md5
|
||||
from urllib.parse import urlencode, unquote
|
||||
from json import loads
|
||||
|
||||
from dateutil.parser import isoparse
|
||||
from babel.dates import format_datetime, format_date, format_time, get_datetime_format
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.data import WIKIDATA_UNITS
|
||||
from searx.network import post, get
|
||||
from searx.utils import searxng_useragent, get_string_replaces_function
|
||||
from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
|
||||
from searx.utils import get_string_replaces_function
|
||||
from searx.external_urls import area_to_osm_zoom
|
||||
from searx.engines.wikipedia import (
|
||||
fetch_wikimedia_traits,
|
||||
get_wiki_params,
|
||||
)
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
from searx.wikidata_properties import (
|
||||
QUERY_TEMPLATE,
|
||||
WDArticle,
|
||||
WDAttrList,
|
||||
WDGeoAttribute,
|
||||
WDImageAttribute,
|
||||
WDURLAttribute,
|
||||
get_attributes,
|
||||
)
|
||||
from searx.wikidata import SPARQL_ENDPOINT_URL, SPARQL_EXPLAIN_URL, get_wikidata_headers
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
@@ -47,78 +53,6 @@ display_type = ["infobox"]
|
||||
one will add a hit to the result list. The first one will show a hit in the
|
||||
info box. Both values can be set, or one of the two can be set."""
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
|
||||
seconds."""
|
||||
|
||||
# SPARQL
|
||||
SPARQL_ENDPOINT_URL = "https://query.wikidata.org/sparql"
|
||||
SPARQL_EXPLAIN_URL = "https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain"
|
||||
WDPType = dict[str | tuple[str, str], str]
|
||||
WIKIDATA_PROPERTIES: WDPType = {
|
||||
"P434": "MusicBrainz",
|
||||
"P435": "MusicBrainz",
|
||||
"P436": "MusicBrainz",
|
||||
"P966": "MusicBrainz",
|
||||
"P345": "IMDb",
|
||||
"P2397": "YouTube",
|
||||
"P1651": "YouTube",
|
||||
"P2002": "Twitter",
|
||||
"P2013": "Facebook",
|
||||
"P2003": "Instagram",
|
||||
"P4033": "Mastodon",
|
||||
"P11947": "Lemmy",
|
||||
"P12622": "PeerTube",
|
||||
}
|
||||
|
||||
# SERVICE wikibase:mwapi : https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual/MWAPI
|
||||
# SERVICE wikibase:label: https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_Label#Manual_Label_SERVICE
|
||||
# https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates
|
||||
# https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format#Data_model
|
||||
# optimization:
|
||||
# * https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/query_optimization
|
||||
# * https://github.com/blazegraph/database/wiki/QueryHints
|
||||
QUERY_TEMPLATE = """
|
||||
SELECT ?item ?itemLabel ?itemDescription ?lat ?long %SELECT%
|
||||
WHERE
|
||||
{
|
||||
SERVICE wikibase:mwapi {
|
||||
bd:serviceParam wikibase:endpoint "www.wikidata.org";
|
||||
wikibase:api "EntitySearch";
|
||||
wikibase:limit 1;
|
||||
mwapi:search "%QUERY%";
|
||||
mwapi:language "%LANGUAGE%".
|
||||
?item wikibase:apiOutputItem mwapi:item.
|
||||
}
|
||||
hint:Prior hint:runFirst "true".
|
||||
|
||||
%WHERE%
|
||||
|
||||
SERVICE wikibase:label {
|
||||
bd:serviceParam wikibase:language "%LANGUAGE%,en".
|
||||
?item rdfs:label ?itemLabel .
|
||||
?item schema:description ?itemDescription .
|
||||
%WIKIBASE_LABELS%
|
||||
}
|
||||
|
||||
}
|
||||
GROUP BY ?item ?itemLabel ?itemDescription ?lat ?long %GROUP_BY%
|
||||
"""
|
||||
|
||||
# Get the calendar names and the property names
|
||||
QUERY_PROPERTY_NAMES = """
|
||||
SELECT ?item ?name
|
||||
WHERE {
|
||||
{
|
||||
SELECT ?item
|
||||
WHERE { ?item wdt:P279* wd:Q12132 }
|
||||
} UNION {
|
||||
VALUES ?item { %ATTRIBUTES% }
|
||||
}
|
||||
OPTIONAL { ?item rdfs:label ?name. }
|
||||
}
|
||||
"""
|
||||
|
||||
# see the property "dummy value" of https://www.wikidata.org/wiki/Q2013 (Wikidata)
|
||||
# hard coded here to avoid to an additional SPARQL request when the server starts
|
||||
DUMMY_ENTITY_URLS = set(
|
||||
@@ -130,357 +64,13 @@ DUMMY_ENTITY_URLS = set(
|
||||
# https://lists.w3.org/Archives/Public/public-rdf-dawg/2011OctDec/0175.html
|
||||
sparql_string_escape = get_string_replaces_function(
|
||||
# fmt: off
|
||||
{
|
||||
"\t": "\\\t",
|
||||
"\n": "\\\n",
|
||||
"\r": "\\\r",
|
||||
"\b": "\\\b",
|
||||
"\f": "\\\f",
|
||||
"\"": "\\\"",
|
||||
"\'": "\\\'",
|
||||
"\\": "\\\\"
|
||||
}
|
||||
{"\t": "\\\t", "\n": "\\\n", "\r": "\\\r", "\b": "\\\b", "\f": "\\\f", "\"": "\\\"", "'": "\\'", "\\": "\\\\"}
|
||||
# fmt: on
|
||||
)
|
||||
|
||||
replace_http_by_https = get_string_replaces_function({"http:": "https:"})
|
||||
|
||||
|
||||
class WDAttribute:
|
||||
|
||||
def __init__(self, name: str):
|
||||
self.name: str = name
|
||||
|
||||
def get_select(self):
|
||||
return "(group_concat(distinct ?{name};separator=', ') as ?{name}s)".replace("{name}", self.name)
|
||||
|
||||
def get_label(self, language: str):
|
||||
return get_label_for_entity(self.name, language)
|
||||
|
||||
def get_where(self):
|
||||
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace("{name}", self.name)
|
||||
|
||||
def get_wikibase_label(self) -> str:
|
||||
return ""
|
||||
|
||||
def get_group_by(self) -> str:
|
||||
return ""
|
||||
|
||||
def get_str(self, result: dict[str, t.Any], language: str) -> str | None: # pylint: disable=unused-argument
|
||||
return result.get(self.name + "s")
|
||||
|
||||
def __repr__(self):
|
||||
return "<" + str(type(self).__name__) + ":" + self.name + ">"
|
||||
|
||||
|
||||
class WDAmountAttribute(WDAttribute):
|
||||
def get_select(self) -> str:
|
||||
return "?{name} ?{name}Unit".replace("{name}", self.name)
|
||||
|
||||
def get_where(self):
|
||||
return """ OPTIONAL { ?item p:{name} ?{name}Node .
|
||||
?{name}Node rdf:type wikibase:BestRank ; ps:{name} ?{name} .
|
||||
OPTIONAL { ?{name}Node psv:{name}/wikibase:quantityUnit ?{name}Unit. } }""".replace(
|
||||
'{name}', self.name
|
||||
)
|
||||
|
||||
def get_group_by(self) -> str:
|
||||
return self.get_select()
|
||||
|
||||
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
|
||||
value: str | None = result.get(self.name)
|
||||
unit: str | None = result.get(self.name + "Unit")
|
||||
if unit is not None:
|
||||
unit = unit.replace("http://www.wikidata.org/entity/", "")
|
||||
return str(value) + " " + get_label_for_entity(unit, language)
|
||||
return value
|
||||
|
||||
|
||||
class WDArticle(WDAttribute):
|
||||
|
||||
def __init__(self, language: str, kwargs: dict[str, t.Any] | None = None):
|
||||
super().__init__("wikipedia")
|
||||
self.language: str = language
|
||||
self.kwargs: dict[str, t.Any] = kwargs or {}
|
||||
|
||||
def get_label(self, language: str):
|
||||
# language parameter is ignored
|
||||
return "Wikipedia ({language})".replace("{language}", self.language)
|
||||
|
||||
def get_select(self):
|
||||
return "?article{language} ?articleName{language}".replace("{language}", self.language)
|
||||
|
||||
def get_where(self):
|
||||
return """OPTIONAL { ?article{language} schema:about ?item ;
|
||||
schema:inLanguage "{language}" ;
|
||||
schema:isPartOf <https://{language}.wikipedia.org/> ;
|
||||
schema:name ?articleName{language} . }""".replace(
|
||||
'{language}', self.language
|
||||
)
|
||||
|
||||
def get_group_by(self):
|
||||
return self.get_select()
|
||||
|
||||
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
|
||||
key = "article{language}".replace("{language}", self.language)
|
||||
return result.get(key)
|
||||
|
||||
|
||||
class WDLabelAttribute(WDAttribute):
|
||||
def get_select(self):
|
||||
return "(group_concat(distinct ?{name}Label;separator=', ') as ?{name}Labels)".replace("{name}", self.name)
|
||||
|
||||
def get_where(self):
|
||||
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace("{name}", self.name)
|
||||
|
||||
def get_wikibase_label(self) -> str:
|
||||
return "?{name} rdfs:label ?{name}Label .".replace("{name}", self.name)
|
||||
|
||||
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
|
||||
return result.get(self.name + "Labels")
|
||||
|
||||
|
||||
class WDURLAttribute(WDAttribute):
|
||||
|
||||
HTTP_WIKIMEDIA_IMAGE: str = "http://commons.wikimedia.org/wiki/Special:FilePath/"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
url_id: str | None = None,
|
||||
url_path_prefix: str | None = None,
|
||||
kwargs: dict[str, t.Any] | None = None,
|
||||
):
|
||||
"""
|
||||
:param url_id: ID matching one key in ``external_urls.json`` for
|
||||
converting IDs to full URLs.
|
||||
|
||||
:param url_path_prefix: Path prefix if the values are of format
|
||||
``account@domain``. If provided, value are rewritten to
|
||||
``https://<domain><url_path_prefix><account>``. For example::
|
||||
|
||||
WDURLAttribute('P4033', url_path_prefix='/@')
|
||||
|
||||
Adds Property `P4033 <https://www.wikidata.org/wiki/Property:P4033>`_
|
||||
to the wikidata query. This field might return for example
|
||||
``libreoffice@fosstodon.org`` and the URL built from this is then:
|
||||
|
||||
- account: ``libreoffice``
|
||||
- domain: ``fosstodon.org``
|
||||
- result url: https://fosstodon.org/@libreoffice
|
||||
"""
|
||||
|
||||
super().__init__(name)
|
||||
self.url_id: str | None = url_id
|
||||
self.url_path_prefix: str | None = url_path_prefix
|
||||
self.kwargs: dict[str, t.Any] = kwargs or {}
|
||||
|
||||
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
|
||||
value: str | None = result.get(self.name + "s")
|
||||
if not value:
|
||||
return None
|
||||
|
||||
value = value.split(",")[0]
|
||||
if self.url_id:
|
||||
url_id = self.url_id
|
||||
if value.startswith(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE):
|
||||
value = value[len(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE) :]
|
||||
url_id = "wikimedia_image"
|
||||
return get_external_url(url_id, value)
|
||||
|
||||
if self.url_path_prefix:
|
||||
[account, domain] = [x.strip("@ ") for x in value.rsplit("@", 1)]
|
||||
return f"https://{domain}{self.url_path_prefix}{account}"
|
||||
|
||||
return value
|
||||
|
||||
|
||||
class WDGeoAttribute(WDAttribute):
|
||||
def get_label(self, language: str):
|
||||
return "OpenStreetMap"
|
||||
|
||||
def get_select(self):
|
||||
return "?{name}Lat ?{name}Long".replace("{name}", self.name)
|
||||
|
||||
def get_where(self):
|
||||
return """OPTIONAL { ?item p:{name}/psv:{name} [
|
||||
wikibase:geoLatitude ?{name}Lat ;
|
||||
wikibase:geoLongitude ?{name}Long ] }""".replace(
|
||||
'{name}', self.name
|
||||
)
|
||||
|
||||
def get_group_by(self):
|
||||
return self.get_select()
|
||||
|
||||
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
|
||||
latitude: str | None = result.get(self.name + "Lat")
|
||||
longitude: str | None = result.get(self.name + "Long")
|
||||
if latitude and longitude:
|
||||
return latitude + " " + longitude
|
||||
return None
|
||||
|
||||
def get_geo_url(self, result: dict[str, t.Any], osm_zoom: int = 19) -> str | None:
|
||||
latitude: str | None = result.get(self.name + "Lat")
|
||||
longitude: str | None = result.get(self.name + "Long")
|
||||
if latitude and longitude:
|
||||
return get_earth_coordinates_url(latitude, longitude, osm_zoom)
|
||||
return None
|
||||
|
||||
|
||||
class WDImageAttribute(WDURLAttribute):
|
||||
|
||||
def __init__(self, name: str, url_id: str | None = None, priority: int = 100):
|
||||
super().__init__(name, url_id)
|
||||
self.priority: int = priority
|
||||
|
||||
|
||||
class WDDateAttribute(WDAttribute):
|
||||
def get_select(self):
|
||||
return "?{name} ?{name}timePrecision ?{name}timeZone ?{name}timeCalendar".replace("{name}", self.name)
|
||||
|
||||
def get_where(self):
|
||||
# To remove duplicate, add
|
||||
# FILTER NOT EXISTS { ?item p:{name}/psv:{name}/wikibase:timeValue ?{name}bis FILTER (?{name}bis < ?{name}) }
|
||||
# this filter is too slow, so the response function ignore duplicate results
|
||||
# (see the seen_entities variable)
|
||||
return """OPTIONAL { ?item p:{name}/psv:{name} [
|
||||
wikibase:timeValue ?{name} ;
|
||||
wikibase:timePrecision ?{name}timePrecision ;
|
||||
wikibase:timeTimezone ?{name}timeZone ;
|
||||
wikibase:timeCalendarModel ?{name}timeCalendar ] . }
|
||||
hint:Prior hint:rangeSafe true;""".replace(
|
||||
'{name}', self.name
|
||||
)
|
||||
|
||||
def get_group_by(self):
|
||||
return self.get_select()
|
||||
|
||||
def format_8(self, value: str, locale: str) -> str: # pylint: disable=unused-argument
|
||||
# precision: less than a year
|
||||
return value
|
||||
|
||||
def format_9(self, value: str, locale: str) -> str:
|
||||
year = int(value)
|
||||
# precision: year
|
||||
if year < 1584:
|
||||
if year < 0:
|
||||
return str(year - 1)
|
||||
return str(year)
|
||||
timestamp = isoparse(value)
|
||||
return format_date(timestamp, format="yyyy", locale=locale)
|
||||
|
||||
def format_10(self, value: str, locale: str) -> str:
|
||||
# precision: month
|
||||
timestamp = isoparse(value)
|
||||
return format_date(timestamp, format="MMMM y", locale=locale)
|
||||
|
||||
def format_11(self, value: str, locale: str) -> str:
|
||||
# precision: day
|
||||
timestamp = isoparse(value)
|
||||
return format_date(timestamp, format="full", locale=locale)
|
||||
|
||||
def format_13(self, value: str, locale: str) -> str:
|
||||
timestamp = isoparse(value)
|
||||
# precision: minute
|
||||
return (
|
||||
get_datetime_format(format, locale=locale)
|
||||
.replace("'", "")
|
||||
.replace("{0}", format_time(timestamp, "full", tzinfo=None, locale=locale))
|
||||
.replace("{1}", format_date(timestamp, "short", locale=locale))
|
||||
)
|
||||
|
||||
def format_14(self, value: str, locale: str) -> str:
|
||||
# precision: second.
|
||||
return format_datetime(isoparse(value), format="full", locale=locale)
|
||||
|
||||
DATE_FORMAT: dict[str, tuple[str, int]] = {
|
||||
"0": ("format_8", 1000000000),
|
||||
"1": ("format_8", 100000000),
|
||||
"2": ("format_8", 10000000),
|
||||
"3": ("format_8", 1000000),
|
||||
"4": ("format_8", 100000),
|
||||
"5": ("format_8", 10000),
|
||||
"6": ("format_8", 1000),
|
||||
"7": ("format_8", 100),
|
||||
"8": ("format_8", 10),
|
||||
"9": ("format_9", 1), # year
|
||||
"10": ("format_10", 1), # month
|
||||
"11": ("format_11", 0), # day
|
||||
"12": ("format_13", 0), # hour (not supported by babel, display minute)
|
||||
"13": ("format_13", 0), # minute
|
||||
"14": ("format_14", 0), # second
|
||||
}
|
||||
|
||||
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
|
||||
value: str | None = result.get(self.name)
|
||||
if value == "" or value is None:
|
||||
return None
|
||||
_p: str = result.get(self.name + "timePrecision") or "1"
|
||||
date_format = WDDateAttribute.DATE_FORMAT.get(_p)
|
||||
if date_format is not None:
|
||||
format_method = getattr(self, date_format[0])
|
||||
precision: int = date_format[1]
|
||||
try:
|
||||
if precision >= 1:
|
||||
_t = value.split("-")
|
||||
if value.startswith("-"):
|
||||
value = "-" + _t[1]
|
||||
else:
|
||||
value = _t[0]
|
||||
return format_method(value, language)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return value
|
||||
return value
|
||||
|
||||
|
||||
WDAttrType = (
|
||||
WDAttribute
|
||||
| WDAmountAttribute
|
||||
| WDArticle
|
||||
| WDLabelAttribute
|
||||
| WDURLAttribute
|
||||
| WDGeoAttribute
|
||||
| WDImageAttribute
|
||||
| WDDateAttribute
|
||||
)
|
||||
WDAttrList = list[WDAttrType]
|
||||
|
||||
|
||||
def get_headers() -> dict[str, str]:
|
||||
# user agent: https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits
|
||||
return {
|
||||
"Accept": "application/sparql-results+json",
|
||||
"User-Agent": f"wikidata engine - {searxng_useragent()}",
|
||||
}
|
||||
|
||||
|
||||
def get_label_for_entity(entity_id: str, language: str) -> str:
|
||||
name = WIKIDATA_PROPERTIES.get(entity_id)
|
||||
if name is None:
|
||||
name = WIKIDATA_PROPERTIES.get((entity_id, language))
|
||||
if name is None:
|
||||
name = WIKIDATA_PROPERTIES.get((entity_id, language.split("-")[0]))
|
||||
if name is None:
|
||||
name = WIKIDATA_PROPERTIES.get((entity_id, "en"))
|
||||
if name is None:
|
||||
name = entity_id
|
||||
return name
|
||||
|
||||
|
||||
def send_wikidata_query(query: str, method: str = "GET", **kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
|
||||
if method == "GET":
|
||||
# query will be cached by wikidata
|
||||
http_response = get(SPARQL_ENDPOINT_URL + "?" + urlencode({"query": query}), headers=get_headers(), **kwargs)
|
||||
else:
|
||||
# query won't be cached by wikidata
|
||||
http_response = post(SPARQL_ENDPOINT_URL, data={"query": query}, headers=get_headers(), **kwargs)
|
||||
if http_response.status_code != 200:
|
||||
logger.debug("SPARQL endpoint error %s", http_response.content.decode())
|
||||
logger.debug("request time %s", str(http_response.elapsed))
|
||||
http_response.raise_for_status()
|
||||
return loads(http_response.content.decode())
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
attributes: WDAttrList
|
||||
@@ -491,7 +81,7 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
params["method"] = "POST"
|
||||
params["url"] = SPARQL_ENDPOINT_URL
|
||||
params["data"] = {"query": query}
|
||||
params["headers"] = get_headers()
|
||||
params["headers"] = get_wikidata_headers()
|
||||
|
||||
# additional parameters (not a part of OnlineParams)
|
||||
params["language"] = eng_tag # type: ignore
|
||||
@@ -584,7 +174,6 @@ def get_results(
|
||||
for attribute in attributes:
|
||||
value: str | None = attribute.get_str(attribute_result, language)
|
||||
if value is not None and value != "":
|
||||
|
||||
if isinstance(attribute, (WDURLAttribute, WDArticle)):
|
||||
# get_select() method : there is group_concat(distinct ...;separator=", ")
|
||||
# split the value here
|
||||
@@ -670,212 +259,15 @@ def get_query(query: str, language: str) -> tuple[str, WDAttrList]:
|
||||
return query, attributes
|
||||
|
||||
|
||||
def get_attributes(language: str):
|
||||
# pylint: disable=too-many-statements
|
||||
attributes: WDAttrList = []
|
||||
|
||||
def add_value(name: str):
|
||||
attributes.append(WDAttribute(name))
|
||||
|
||||
def add_amount(name: str):
|
||||
attributes.append(WDAmountAttribute(name))
|
||||
|
||||
def add_label(name: str):
|
||||
attributes.append(WDLabelAttribute(name))
|
||||
|
||||
def add_url(name: str, url_id: str | None = None, url_path_prefix: str | None = None, **kwargs: dict[str, t.Any]):
|
||||
attributes.append(WDURLAttribute(name, url_id, url_path_prefix, kwargs))
|
||||
|
||||
def add_image(name: str, url_id: str | None = None, priority: int = 1):
|
||||
attributes.append(WDImageAttribute(name, url_id, priority))
|
||||
|
||||
def add_date(name: str):
|
||||
attributes.append(WDDateAttribute(name))
|
||||
|
||||
# Dates
|
||||
for p in [
|
||||
"P571", # inception date
|
||||
"P576", # dissolution date
|
||||
"P580", # start date
|
||||
"P582", # end date
|
||||
"P569", # date of birth
|
||||
"P570", # date of death
|
||||
"P619", # date of spacecraft launch
|
||||
"P620",
|
||||
]: # date of spacecraft landing
|
||||
add_date(p)
|
||||
|
||||
for p in [
|
||||
"P27", # country of citizenship
|
||||
"P495", # country of origin
|
||||
"P17", # country
|
||||
"P159",
|
||||
]: # headquarters location
|
||||
add_label(p)
|
||||
|
||||
# Places
|
||||
for p in [
|
||||
"P36", # capital
|
||||
"P35", # head of state
|
||||
"P6", # head of government
|
||||
"P122", # basic form of government
|
||||
"P37",
|
||||
]: # official language
|
||||
add_label(p)
|
||||
|
||||
add_value("P1082") # population
|
||||
add_amount("P2046") # area
|
||||
add_amount("P281") # postal code
|
||||
add_label("P38") # currency
|
||||
add_amount("P2048") # height (building)
|
||||
|
||||
# Media
|
||||
for p in [
|
||||
"P400", # platform (videogames, computing)
|
||||
"P50", # author
|
||||
"P170", # creator
|
||||
"P57", # director
|
||||
"P175", # performer
|
||||
"P178", # developer
|
||||
"P162", # producer
|
||||
"P176", # manufacturer
|
||||
"P58", # screenwriter
|
||||
"P272", # production company
|
||||
"P264", # record label
|
||||
"P123", # publisher
|
||||
"P449", # original network
|
||||
"P750", # distributed by
|
||||
"P86",
|
||||
]: # composer
|
||||
add_label(p)
|
||||
|
||||
add_date("P577") # publication date
|
||||
add_label("P136") # genre (music, film, artistic...)
|
||||
add_label("P364") # original language
|
||||
add_value("P212") # ISBN-13
|
||||
add_value("P957") # ISBN-10
|
||||
add_label("P275") # copyright license
|
||||
add_label("P277") # programming language
|
||||
add_value("P348") # version
|
||||
add_label("P840") # narrative location
|
||||
|
||||
# Languages
|
||||
add_value("P1098") # number of speakers
|
||||
add_label("P282") # writing system
|
||||
add_label("P1018") # language regulatory body
|
||||
add_value("P218") # language code (ISO 639-1)
|
||||
|
||||
# Other
|
||||
add_label("P169") # ceo
|
||||
add_label("P112") # founded by
|
||||
add_label("P1454") # legal form (company, organization)
|
||||
add_label("P137") # operator (service, facility, ...)
|
||||
add_label("P1029") # crew members (tripulation)
|
||||
add_label("P225") # taxon name
|
||||
add_value("P274") # chemical formula
|
||||
add_label("P1346") # winner (sports, contests, ...)
|
||||
add_value("P1120") # number of deaths
|
||||
add_value("P498") # currency code (ISO 4217)
|
||||
|
||||
# URL
|
||||
kwargs: dict[str, t.Any] = {"official": True}
|
||||
add_url("P856", **kwargs) # official website
|
||||
attributes.append(WDArticle(language)) # wikipedia (user language)
|
||||
if not language.startswith("en"):
|
||||
attributes.append(WDArticle("en")) # wikipedia (english)
|
||||
|
||||
add_url("P1324") # source code repository
|
||||
add_url("P1581") # blog
|
||||
add_url("P434", url_id="musicbrainz_artist")
|
||||
add_url("P435", url_id="musicbrainz_work")
|
||||
add_url("P436", url_id="musicbrainz_release_group")
|
||||
add_url("P966", url_id="musicbrainz_label")
|
||||
add_url("P345", url_id="imdb_id")
|
||||
add_url("P2397", url_id="youtube_channel")
|
||||
add_url("P1651", url_id="youtube_video")
|
||||
add_url("P2002", url_id="twitter_profile")
|
||||
add_url("P2013", url_id="facebook_profile")
|
||||
add_url("P2003", url_id="instagram_profile")
|
||||
|
||||
# Fediverse
|
||||
add_url("P4033", url_path_prefix="/@") # Mastodon user
|
||||
add_url("P11947", url_path_prefix="/c/") # Lemmy community
|
||||
add_url("P12622", url_path_prefix="/c/") # PeerTube channel
|
||||
|
||||
# Map
|
||||
attributes.append(WDGeoAttribute("P625"))
|
||||
|
||||
# Image
|
||||
add_image("P15", priority=1, url_id="wikimedia_image") # route map
|
||||
add_image("P242", priority=2, url_id="wikimedia_image") # locator map
|
||||
add_image("P154", priority=3, url_id="wikimedia_image") # logo
|
||||
add_image("P18", priority=4, url_id="wikimedia_image") # image
|
||||
add_image("P41", priority=5, url_id="wikimedia_image") # flag
|
||||
add_image("P2716", priority=6, url_id="wikimedia_image") # collage
|
||||
add_image("P2910", priority=7, url_id="wikimedia_image") # icon
|
||||
|
||||
return attributes
|
||||
|
||||
|
||||
def debug_explain_wikidata_query(query: str, method: str = "GET"):
|
||||
if method == "GET":
|
||||
http_response = get(SPARQL_EXPLAIN_URL + "&" + urlencode({"query": query}), headers=get_headers())
|
||||
http_response = get(SPARQL_EXPLAIN_URL + "&" + urlencode({"query": query}), headers=get_wikidata_headers())
|
||||
else:
|
||||
http_response = post(SPARQL_EXPLAIN_URL, data={"query": query}, headers=get_headers())
|
||||
http_response = post(SPARQL_EXPLAIN_URL, data={"query": query}, headers=get_wikidata_headers())
|
||||
http_response.raise_for_status()
|
||||
return http_response.content
|
||||
|
||||
|
||||
def init(_):
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache("wikidata")
|
||||
|
||||
# In an environment with competing processes, the initial loading of the
|
||||
# cache is required only once.
|
||||
eng_state: str | None = CACHE.get("eng_state")
|
||||
if not eng_state or not eng_state.startswith("STATE:"):
|
||||
CACHE.set("eng_state", f"STATE: being initialized by PID {os.getpid()}")
|
||||
try:
|
||||
init_wikidata_properties()
|
||||
except Exception:
|
||||
CACHE.set("eng_state", f"ERROR: initialization by PID {os.getpid()} failed.")
|
||||
raise
|
||||
else:
|
||||
logger.debug(eng_state)
|
||||
|
||||
|
||||
def init_wikidata_properties():
|
||||
global WIKIDATA_PROPERTIES # pylint: disable=global-statement
|
||||
p: WDPType = CACHE.get(key="WIKIDATA_PROPERTIES")
|
||||
if p:
|
||||
WIKIDATA_PROPERTIES = p
|
||||
return
|
||||
|
||||
# WIKIDATA_PROPERTIES : add unit symbols
|
||||
for k, v in WIKIDATA_UNITS.items():
|
||||
WIKIDATA_PROPERTIES[k] = v["symbol"]
|
||||
|
||||
# WIKIDATA_PROPERTIES : add property labels
|
||||
wikidata_property_names: list[str] = []
|
||||
for attribute in get_attributes("en"):
|
||||
if type(attribute) in (WDAttribute, WDAmountAttribute, WDURLAttribute, WDDateAttribute, WDLabelAttribute):
|
||||
if attribute.name not in WIKIDATA_PROPERTIES:
|
||||
wikidata_property_names.append("wd:" + attribute.name)
|
||||
query = QUERY_PROPERTY_NAMES.replace("%ATTRIBUTES%", " ".join(wikidata_property_names))
|
||||
kwargs: dict[str, t.Any] = {"timeout": 20}
|
||||
jsonresponse = send_wikidata_query(query, **kwargs)
|
||||
for result in jsonresponse.get("results", {}).get("bindings", {}):
|
||||
name_field = result.get("name")
|
||||
if not name_field:
|
||||
continue
|
||||
name = name_field["value"]
|
||||
lang = name_field["xml:lang"]
|
||||
entity_id = result["item"]["value"].replace("http://www.wikidata.org/entity/", "")
|
||||
WIKIDATA_PROPERTIES[(entity_id, lang)] = name.capitalize()
|
||||
|
||||
CACHE.set(key="WIKIDATA_PROPERTIES", value=WIKIDATA_PROPERTIES)
|
||||
|
||||
|
||||
def fetch_traits(engine_traits: EngineTraits):
|
||||
"""Uses languages evaluated from :py:obj:`wikipedia.fetch_wikimedia_traits
|
||||
<searx.engines.wikipedia.fetch_wikimedia_traits>` and removes
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
Wolfram|Alpha (Science)
|
||||
"""
|
||||
|
||||
|
||||
from json import loads
|
||||
from urllib.parse import urlencode
|
||||
|
||||
@@ -53,7 +52,7 @@ seconds."""
|
||||
|
||||
def init(engine_settings):
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"]) # type:ignore
|
||||
CACHE = EngineCache(engine_settings["name"]) # type: ignore
|
||||
|
||||
|
||||
def obtain_token() -> str:
|
||||
|
||||
@@ -50,6 +50,7 @@ the engine).
|
||||
Implementations
|
||||
===============
|
||||
"""
|
||||
|
||||
# pylint: disable=fixme
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ from lxml import html
|
||||
from searx.exceptions import SearxEngineCaptchaException
|
||||
from searx.utils import humanize_bytes, eval_xpath, eval_xpath_list, extract_text, extr
|
||||
|
||||
|
||||
# Engine metadata
|
||||
about = {
|
||||
"website": 'https://yandex.com/',
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
:members:
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
|
||||
__all__ = ["SXNG_Request", "sxng_request", "SXNG_Response"]
|
||||
|
||||
@@ -5,7 +5,6 @@ import math
|
||||
|
||||
from searx.data import EXTERNAL_URLS
|
||||
|
||||
|
||||
IMDB_PREFIX_TO_URL_ID = {
|
||||
'tt': 'imdb_title',
|
||||
'mn': 'imdb_name',
|
||||
|
||||
@@ -8,7 +8,6 @@ an example in which the command line is called in the development environment::
|
||||
(py3) python -m searx.favicons --help
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["init", "favicon_url", "favicon_proxy"]
|
||||
|
||||
import pathlib
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Implementations for a favicon proxy"""
|
||||
|
||||
|
||||
from typing import Callable
|
||||
|
||||
import importlib
|
||||
|
||||
@@ -6,7 +6,6 @@ timeout``) and returns a tuple ``(data, mime)``.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["DEFAULT_RESOLVER_MAP", "allesedv", "duckduckgo", "google", "kagi", "yandex"]
|
||||
|
||||
from typing import Callable
|
||||
|
||||
@@ -36,7 +36,6 @@ from .. import get_setting
|
||||
from ..version import GIT_URL
|
||||
from ..locales import LOCALE_NAMES
|
||||
|
||||
|
||||
logger = logging.getLogger('searx.infopage')
|
||||
_INFO_FOLDER = os.path.abspath(os.path.dirname(__file__))
|
||||
INFO_PAGES: 'InfoPageSet'
|
||||
|
||||
@@ -26,7 +26,6 @@ SearXNG’s locale implementations
|
||||
================================
|
||||
"""
|
||||
|
||||
|
||||
import typing as t
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ from searx.exceptions import (
|
||||
from searx import searx_parent_dir, settings
|
||||
from searx.engines import engines
|
||||
|
||||
|
||||
errors_per_engines: dict[str, t.Any] = {}
|
||||
|
||||
LogParametersType = tuple[str, ...]
|
||||
|
||||
@@ -8,7 +8,6 @@ import threading
|
||||
|
||||
from searx import logger
|
||||
|
||||
|
||||
__all__ = ["Histogram", "HistogramStorage", "CounterStorage"]
|
||||
|
||||
logger = logger.getChild('searx.metrics')
|
||||
|
||||
@@ -20,7 +20,6 @@ from searx.extended_types import SXNG_Response
|
||||
from .client import new_client, get_loop, AsyncHTTPTransportNoHttp
|
||||
from .raise_for_httperror import raise_for_httperror
|
||||
|
||||
|
||||
logger = logger.getChild('network')
|
||||
DEFAULT_NAME = '__DEFAULT__'
|
||||
NETWORKS: dict[str, "Network"] = {}
|
||||
|
||||
@@ -94,7 +94,6 @@ Implementation
|
||||
:members:
|
||||
"""
|
||||
|
||||
|
||||
__all__ = ["PluginInfo", "Plugin", "PluginStorage", "PluginCfg"]
|
||||
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ user searches for ``tor-check``. It fetches the tor exit node list from
|
||||
:py:obj:`url_exit_list` and parses all the IPs into a list, then checks if the
|
||||
user's IP address is in it.
|
||||
"""
|
||||
|
||||
from ipaddress import ip_address
|
||||
import typing
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ converters, each converter is one item in the list (compare
|
||||
of measurement are evaluated. The weighting in the evaluation results from the
|
||||
sorting of the :py:obj:`list of unit converters<symbol_to_si>`.
|
||||
"""
|
||||
|
||||
import typing
|
||||
import re
|
||||
import babel.numbers
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
import typing as t
|
||||
|
||||
from base64 import urlsafe_b64encode, urlsafe_b64decode
|
||||
from zlib import compress, decompress
|
||||
from zlib import compress, decompressobj
|
||||
from urllib.parse import parse_qs, urlencode
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
@@ -517,7 +517,7 @@ class Preferences:
|
||||
|
||||
def parse_encoded_data(self, input_data: str):
|
||||
"""parse (base64) preferences from request (``flask.request.form['preferences']``)"""
|
||||
bin_data = decompress(urlsafe_b64decode(input_data))
|
||||
bin_data = decompressobj().decompress(urlsafe_b64decode(input_data), 16 * 1024)
|
||||
dict_data = {}
|
||||
for x, y in parse_qs(bin_data.decode('ascii'), keep_blank_values=True).items():
|
||||
dict_data[x] = y[0]
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
gradually. For more, please read :ref:`result types`.
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ template.
|
||||
:members:
|
||||
:show-inheritance:
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ template. For highlighting the code passages, Pygments_ is used.
|
||||
:show-inheritance:
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods, disable=invalid-name
|
||||
|
||||
__all__ = ["Code"]
|
||||
@@ -26,7 +27,6 @@ from pygments.formatters import HtmlFormatter # pylint: disable=no-name-in-modu
|
||||
|
||||
from ._base import MainResult
|
||||
|
||||
|
||||
_pygments_languages: list[str] = []
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ template.
|
||||
:show-inheritance:
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ template.
|
||||
:members:
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
__all__ = ["Image", "ImageRef"]
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ template.
|
||||
:show-inheritance:
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ Related topics:
|
||||
:show-inheritance:
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods, disable=invalid-name
|
||||
|
||||
__all__ = ["Paper"]
|
||||
|
||||
@@ -8,7 +8,6 @@ from searx import webutils
|
||||
from searx import engines
|
||||
from searx.weather import WeatherConditionType
|
||||
|
||||
|
||||
__all__ = [
|
||||
'CONSTANT_NAMES',
|
||||
'CATEGORY_NAMES',
|
||||
|
||||
@@ -1217,12 +1217,12 @@ engines:
|
||||
- name: google
|
||||
engine: google
|
||||
shortcut: go
|
||||
inactive: true
|
||||
disabled: true
|
||||
|
||||
- name: google images
|
||||
engine: google_images
|
||||
shortcut: goi
|
||||
inactive: true
|
||||
disabled: true
|
||||
|
||||
- name: google news
|
||||
engine: google_news
|
||||
@@ -1231,7 +1231,6 @@ engines:
|
||||
- name: google videos
|
||||
engine: google_videos
|
||||
shortcut: gov
|
||||
inactive: true
|
||||
|
||||
- name: google cse
|
||||
engine: google_cse
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Implementation of the default settings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing as t
|
||||
|
||||
2
searx/static/themes/simple/sxng-ltr.min.css
vendored
2
searx/static/themes/simple/sxng-ltr.min.css
vendored
File diff suppressed because one or more lines are too long
2
searx/static/themes/simple/sxng-rtl.min.css
vendored
2
searx/static/themes/simple/sxng-rtl.min.css
vendored
File diff suppressed because one or more lines are too long
Binary file not shown.
@@ -24,7 +24,7 @@ msgstr ""
|
||||
"Project-Id-Version: searx\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2026-07-15 15:45+0000\n"
|
||||
"PO-Revision-Date: 2026-07-17 12:20+0000\n"
|
||||
"PO-Revision-Date: 2026-08-21 20:54+0000\n"
|
||||
"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
|
||||
"Language-Team: Bulgarian <https://translate.codeberg.org/projects/searxng/"
|
||||
"searxng/bg/>\n"
|
||||
@@ -33,7 +33,7 @@ msgstr ""
|
||||
"Content-Type: text/plain; charset=utf-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Plural-Forms: nplurals=2; plural=n != 1;\n"
|
||||
"X-Generator: Weblate 2026.6.1\n"
|
||||
"X-Generator: Weblate 2026.8.1\n"
|
||||
"Generated-By: Babel 2.18.0\n"
|
||||
|
||||
#. CONSTANT_NAMES['NO_SUBGROUPING']
|
||||
@@ -737,7 +737,7 @@ msgstr ""
|
||||
|
||||
#: searx/plugins/calculator.py:25
|
||||
msgid "Calculator"
|
||||
msgstr ""
|
||||
msgstr "Калкулатор"
|
||||
|
||||
#: searx/plugins/calculator.py:26
|
||||
msgid "Parses and solves mathematical expressions."
|
||||
@@ -1653,11 +1653,11 @@ msgstr "Резолюция"
|
||||
|
||||
#: searx/templates/simple/result_templates/images.html:55
|
||||
msgid "Image formats"
|
||||
msgstr ""
|
||||
msgstr "формат на изображението"
|
||||
|
||||
#: searx/templates/simple/result_templates/images.html:56
|
||||
msgid "original format"
|
||||
msgstr ""
|
||||
msgstr "оригинален формат"
|
||||
|
||||
#: searx/templates/simple/result_templates/images.html:64
|
||||
msgid "View source"
|
||||
|
||||
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user