2 Commits

Author SHA1 Message Date
Ivan Gabaldon
00eaf1157d fmt 2026-08-21 22:53:43 +02:00
dependabot[bot]
f92003b360 [upd] pypi: Bump black from 25.9.0 to 26.5.1
Bumps [black](https://github.com/psf/black) from 25.9.0 to 26.5.1.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/compare/25.9.0...26.5.1)

---
updated-dependencies:
- dependency-name: black
  dependency-version: 26.5.1
  dependency-type: direct:development
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-21 07:24:43 +00:00
50 changed files with 2806 additions and 25209 deletions

View File

@@ -31,7 +31,7 @@ jobs:
- update_external_bangs.py
- update_firefox_version.py
- update_engine_traits.py
- update_wikidata.py
- update_wikidata_units.py
- update_engine_descriptions.py
permissions:

View File

@@ -2,7 +2,7 @@
/*
this file is generated automatically by searxng_extra/update/update_pygments.py
using pygments version 2.21.0:
using pygments version 2.20.0:
./manage templates.simple.pygments
*/
@@ -114,14 +114,14 @@
.gd { color: #FF4689 } /* Generic.Deleted */
.ge { color: #F8F8F2; font-style: italic } /* Generic.Emph */
.ges { color: #F8F8F2; font-weight: bold; font-style: italic } /* Generic.EmphStrong */
.gr { color: #FF4689 } /* Generic.Error */
.gr { color: #F8F8F2 } /* Generic.Error */
.gh { color: #F8F8F2 } /* Generic.Heading */
.gi { color: #A6E22E } /* Generic.Inserted */
.go { color: #66D9EF } /* Generic.Output */
.gp { color: #FF4689; font-weight: bold } /* Generic.Prompt */
.gs { color: #F8F8F2; font-weight: bold } /* Generic.Strong */
.gu { color: #959077 } /* Generic.Subheading */
.gt { color: #66D9EF } /* Generic.Traceback */
.gt { color: #F8F8F2 } /* Generic.Traceback */
.kc { color: #66D9EF } /* Keyword.Constant */
.kd { color: #66D9EF } /* Keyword.Declaration */
.kn { color: #FF4689 } /* Keyword.Namespace */
@@ -132,7 +132,7 @@
.m { color: #AE81FF } /* Literal.Number */
.s { color: #E6DB74 } /* Literal.String */
.na { color: #A6E22E } /* Name.Attribute */
.nb { color: #A6E22E } /* Name.Builtin */
.nb { color: #F8F8F2 } /* Name.Builtin */
.nc { color: #A6E22E } /* Name.Class */
.no { color: #66D9EF } /* Name.Constant */
.nd { color: #A6E22E } /* Name.Decorator */
@@ -166,7 +166,7 @@
.sr { color: #E6DB74 } /* Literal.String.Regex */
.s1 { color: #E6DB74 } /* Literal.String.Single */
.ss { color: #E6DB74 } /* Literal.String.Symbol */
.bp { color: #A6E22E } /* Name.Builtin.Pseudo */
.bp { color: #F8F8F2 } /* Name.Builtin.Pseudo */
.fm { color: #A6E22E } /* Name.Function.Magic */
.vc { color: #F8F8F2 } /* Name.Variable.Class */
.vg { color: #F8F8F2 } /* Name.Variable.Global */

View File

@@ -15,12 +15,12 @@
"swiped-events": "1.2.0"
},
"devDependencies": {
"@biomejs/biome": "2.5.9",
"@biomejs/biome": "2.5.7",
"@types/node": "^26.2.0",
"browserslist": "^4.28.8",
"browserslist-to-esbuild": "^2.1.1",
"edge.js": "^6.5.1",
"less": "^4.9.0",
"less": "^4.8.1",
"mathjs": "^15.2.0",
"sharp": "~0.35.3",
"sort-package-json": "^4.0.0",
@@ -69,9 +69,9 @@
}
},
"node_modules/@biomejs/biome": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.9.tgz",
"integrity": "sha512-KkgCvdHB4IhtpHpF564plA9jo6fDOwWGQ/3jvreLzgOtRLEDoPqr7QO9qejNA8jKwDsSkAKr77hqBHnyUbIw4g==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.7.tgz",
"integrity": "sha512-zr8K/DcY5tYsQOQwqMJ0AWElo6QgmgNI7idXgXLhevVszlt8RGVpesEJPqx3ThazLaOwjJ5Y8fz3BtH5fGZNsw==",
"dev": true,
"license": "MIT OR Apache-2.0",
"bin": {
@@ -85,20 +85,20 @@
"url": "https://opencollective.com/biome"
},
"optionalDependencies": {
"@biomejs/cli-darwin-arm64": "2.5.9",
"@biomejs/cli-darwin-x64": "2.5.9",
"@biomejs/cli-linux-arm64": "2.5.9",
"@biomejs/cli-linux-arm64-musl": "2.5.9",
"@biomejs/cli-linux-x64": "2.5.9",
"@biomejs/cli-linux-x64-musl": "2.5.9",
"@biomejs/cli-win32-arm64": "2.5.9",
"@biomejs/cli-win32-x64": "2.5.9"
"@biomejs/cli-darwin-arm64": "2.5.7",
"@biomejs/cli-darwin-x64": "2.5.7",
"@biomejs/cli-linux-arm64": "2.5.7",
"@biomejs/cli-linux-arm64-musl": "2.5.7",
"@biomejs/cli-linux-x64": "2.5.7",
"@biomejs/cli-linux-x64-musl": "2.5.7",
"@biomejs/cli-win32-arm64": "2.5.7",
"@biomejs/cli-win32-x64": "2.5.7"
}
},
"node_modules/@biomejs/cli-darwin-arm64": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.9.tgz",
"integrity": "sha512-am22pX2aBqznqq1eMyIj/bZ++riF3Lk6ct7cbv+gQK0csFhr+d8O0RkOi2FF2qSgFgANbqNkIZ0/PxlnW2pLFg==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.7.tgz",
"integrity": "sha512-vxo/Ls3/PYdQWyLhYYcgMOCzQypAjcY+iihS8M0wW03l16TCLW4zqZzGo75gm1VdCMj38hTVZ31KBWrZ4G9dJw==",
"cpu": [
"arm64"
],
@@ -113,9 +113,9 @@
}
},
"node_modules/@biomejs/cli-darwin-x64": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.9.tgz",
"integrity": "sha512-l44KWDHLDvEnD0N/XcrVs7VXb3A18xL7QS3WB0eL93wbmk529ffIG55vleGCqaunpRUjLrdnjK05Qki1dsjylg==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.7.tgz",
"integrity": "sha512-Cd3Ga61amT/Yl/0x8elP5hhGYaFy4bw6WuysTgf7oo8TA5tJ5A1k+DkVoJ2BHbTVil51gTX9VPzArnrlLJ3Kyg==",
"cpu": [
"x64"
],
@@ -130,9 +130,9 @@
}
},
"node_modules/@biomejs/cli-linux-arm64": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.9.tgz",
"integrity": "sha512-ICaK+IYaVZvKbBxX2rwrPT0DdUDMnE9Vm3nQGe+mltQPmUg19pONzkPWGdY4FCsoreDETWDynvdt4ysCbF5gNQ==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.7.tgz",
"integrity": "sha512-rR2QE0yF2GYSuYuKIa7pKvODGJqnOH+2eDREAM8wV+mWKSkMQKdAp4zXEZfTaxY8PMoNONnpgSWcBCyLDPDOKg==",
"cpu": [
"arm64"
],
@@ -150,9 +150,9 @@
}
},
"node_modules/@biomejs/cli-linux-arm64-musl": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.9.tgz",
"integrity": "sha512-7ImVPwBLCtkmpR5esd8RHhTqW94f0JLJQum6AneYcy94jRm18TaPPm7slaigGzFhfgt3QiD1Vj52LKmBAnKizA==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.7.tgz",
"integrity": "sha512-xPI5yB6XlpDbNkS+bm1t42olw5c4l3UrlOmLg7KtLJvjvkNF/1V4tnUgfkylGIeb3u/T+BzMGYqgQhzjAoJzuQ==",
"cpu": [
"arm64"
],
@@ -170,9 +170,9 @@
}
},
"node_modules/@biomejs/cli-linux-x64": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.9.tgz",
"integrity": "sha512-z22Q/zFYSvbIJfW1CbfZPu4X8PddS6Qd2ORbc6h+aT6EcwAxUF3m6fA4HjNvA3TU4X0dTJRwNPB165ES3PJXzg==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.7.tgz",
"integrity": "sha512-FQgqJhscrqJUFptGaRSUJWlXAExwWcDwLuK49dvKfkQ1bB5SEEyFssnsxQY83Xm6jR0EbbX3+8+D5bfvYqUG2Q==",
"cpu": [
"x64"
],
@@ -190,9 +190,9 @@
}
},
"node_modules/@biomejs/cli-linux-x64-musl": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.9.tgz",
"integrity": "sha512-RXGaD0o1/pTTguYw1aeDJh9ad6Lfrui0fI7mBderTyGr7WuUJkBIttgLkR3XJyoxOkkgfBDspaUT8wXArTqLZw==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.7.tgz",
"integrity": "sha512-rE5VZi+qtmPgQH+l7jVxYoZ18b/TiHEhulhMpjmCZH1PltSbjRcxNWywC3HZ9tYottG7ORkeTtoscBilKSBm0g==",
"cpu": [
"x64"
],
@@ -210,9 +210,9 @@
}
},
"node_modules/@biomejs/cli-win32-arm64": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.9.tgz",
"integrity": "sha512-nHK+/HHC+D0ogAHUxomgoSTdjImb6fmNNVTKmf0tyu4eDL1DqPKIHc+i+UL8+b0RnAu8224qo8F2tCVnaT0A3w==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.7.tgz",
"integrity": "sha512-Oq4x0CCwP4jirrcTywXs5kOGZ4v5vuEP+gWrbtjApOA2CL9F3F9GlIdQIci8AKSCa/zURanMRpX/4wQ7Am6hHg==",
"cpu": [
"arm64"
],
@@ -227,9 +227,9 @@
}
},
"node_modules/@biomejs/cli-win32-x64": {
"version": "2.5.9",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.9.tgz",
"integrity": "sha512-Yiq0H56LjXSSw/hd9YkXgSLQfzyDJzbzU2TezozxyNw+uKWAqOtqGVvBfzKRRDiaFF5avGAhHdWKx7LtDOShUw==",
"version": "2.5.7",
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.7.tgz",
"integrity": "sha512-V+0wu/nrj2S+MhP4EQ0uHNolP0IALEsz45pg0WoKkHfDeh0+ItHwP/p7bX5RPoMOl9NkpHYWdYPhIcy2mACHvQ==",
"cpu": [
"x64"
],
@@ -3233,9 +3233,9 @@
"license": "Apache-2.0"
},
"node_modules/less": {
"version": "4.9.0",
"resolved": "https://registry.npmjs.org/less/-/less-4.9.0.tgz",
"integrity": "sha512-umRhrCH7fCi8Uj2RcwKjJdvUORTjeWqkdKx0LbcZvjIwsAVsnIAGcxHaqowPeBFBjQuWOeC/bve0AlpFzF/+SQ==",
"version": "4.8.1",
"resolved": "https://registry.npmjs.org/less/-/less-4.8.1.tgz",
"integrity": "sha512-jQ3lRIo1aUtiWVYXZ7mk4+V4BjCGswF3IxTLJ+4RUta8ZiHh8lhkig2G8dya2eCcyR1dYUvzuV46EkJN8PSwww==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {

View File

@@ -29,12 +29,12 @@
"swiped-events": "1.2.0"
},
"devDependencies": {
"@biomejs/biome": "2.5.9",
"@biomejs/biome": "2.5.7",
"@types/node": "^26.2.0",
"browserslist": "^4.28.8",
"browserslist-to-esbuild": "^2.1.1",
"edge.js": "^6.5.1",
"less": "^4.9.0",
"less": "^4.8.1",
"mathjs": "^15.2.0",
"sharp": "~0.35.3",
"sort-package-json": "^4.0.0",

View File

@@ -80,8 +80,8 @@ same environment, here are a few examples::
# to test one of the update scripts
(dev.env)$ searxng_extra/update/update_engine_traits.py --help
# to test the update of the wikidata units and property names
(dev.env)$ searxng_extra/update/update_wikidata.py
# to test the update of the wikidata units
(dev.env)$ searxng_extra/update/update_wikidata_units.py
.. sidebar:: further read

View File

@@ -90,10 +90,10 @@ Scripts to update static data in :origin:`searx/data/`
:members:
``update_wikidata.py``
``update_wikidata_units.py``
============================
:origin:`[source] <searxng_extra/update/update_wikidata.py>`
:origin:`[source] <searxng_extra/update/update_wikidata_units.py>`
.. automodule:: searxng_extra.update.update_wikidata
.. automodule:: searxng_extra.update.update_wikidata_units
:members:

View File

@@ -23,6 +23,6 @@ coloredlogs==15.0.1
docutils>=0.21.2;python_version <= "3.11"
docutils>=0.22.4; python_version > "3.11"
parameterized==0.9.0
granian[reload]==2.8.2
basedpyright==1.39.10
granian[reload]==2.8.1
basedpyright==1.39.9
types-lxml==2026.2.16

View File

@@ -1,2 +1,2 @@
granian==2.8.2
granian[pname]==2.8.2
granian==2.8.1
granian[pname]==2.8.1

View File

@@ -3,8 +3,8 @@ babel==2.18.0
flask-babel==4.0.0
flask==3.1.3
jinja2==3.1.6
lxml==6.1.2
pygments==2.21.0
lxml==6.1.1
pygments==2.20.0
python-dateutil==2.9.0.post0
pyyaml==6.0.3
httpx[http2]==0.28.1

View File

@@ -127,17 +127,18 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
def google_complete(query: str, sxng_locale: str) -> list[str]:
"""Autocomplete from Google. Supports Google's languages
"""Autocomplete from Google. Supports Google's languages and subdomains
(:py:obj:`searx.engines.google.get_google_info`) by using the async REST
API::
https://www.google.com/complete/search?{args}
https://{subdomain}/complete/search?{args}
"""
data = ENGINE_TRAITS.get("google") or {}
traits = EngineTraits(**data)
google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
url = 'https://{subdomain}/complete/search?{args}'
args = urlencode(
{
'q': query,
@@ -147,7 +148,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
)
results: list[str] = []
resp = get('https://www.google.com/complete/search?' + args)
resp = get(url.format(subdomain=google_info['subdomain'], args=args))
if resp and resp.ok:
json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
data = json.loads(json_txt)

View File

@@ -33,13 +33,6 @@ class WikiDataUnitType(t.TypedDict):
to_si_factor: float
WikiDataPropertyNameType = str | dict[str, str]
"""Name of a Wikidata property. Can be either the plain name or a dictionary of
language code to property name, e.g. ``{"en": "Date of birth"}``."""
WikiDataPropertiesType = dict[str, WikiDataPropertyNameType]
"""Dictionary from wikidata property ID to property name."""
class LocalesType(t.TypedDict):
"""Data structure of an item in ``locales.json``"""
@@ -49,7 +42,6 @@ class LocalesType(t.TypedDict):
USER_AGENTS: UserAgentType
WIKIDATA_UNITS: dict[str, WikiDataUnitType]
WIKIDATA_PROPERTIES: WikiDataPropertiesType
TRACKER_PATTERNS: TrackerPatternsDB
LOCALES: LocalesType
CURRENCIES: CurrenciesDB
@@ -61,12 +53,11 @@ ENGINE_DESCRIPTIONS: dict[str, dict[str, t.Any]]
ENGINE_TRAITS: dict[str, dict[str, t.Any]]
lazy_globals: dict[str, t.Any] = {
lazy_globals = {
"CURRENCIES": CurrenciesDB(),
"USER_AGENTS": None,
"EXTERNAL_URLS": None,
"WIKIDATA_UNITS": None,
"WIKIDATA_PROPERTIES": None,
"EXTERNAL_BANGS": None,
"OSM_KEYS_TAGS": None,
"ENGINE_DESCRIPTIONS": None,
@@ -79,7 +70,6 @@ data_json_files = {
"USER_AGENTS": "useragents.json",
"EXTERNAL_URLS": "external_urls.json",
"WIKIDATA_UNITS": "wikidata_units.json",
"WIKIDATA_PROPERTIES": "wikidata_properties.json",
"EXTERNAL_BANGS": "external_bangs.json",
"OSM_KEYS_TAGS": "osm_keys_tags.json",
"ENGINE_DESCRIPTIONS": "engine_descriptions.json",

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -5,7 +5,7 @@
],
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
"versions": [
"154.0",
"153.0"
"153.0",
"152.0"
]
}

File diff suppressed because it is too large Load Diff

View File

@@ -3474,6 +3474,11 @@
"symbol": "mm⁻²",
"to_si_factor": 1e-06
},
"Q136039973": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q1361854": {
"si_name": "Q11570",
"symbol": "dwt",
@@ -5249,11 +5254,6 @@
"symbol": "μA",
"to_si_factor": 1e-06
},
"Q31274648": {
"si_name": "Q6137407",
"symbol": "FPS",
"to_si_factor": 1.0
},
"Q3186734": {
"si_name": "Q3186734",
"symbol": "J/(m³ K)",

View File

@@ -9,15 +9,12 @@ engines:
- :ref:`google scholar engine`
- :ref:`google autocomplete`
This implementation uses Nokia user agents to request an XML layout from Google.
The normal web version requires executing JavaScript to load the results and
therefore is currently not used here. See `Google discussion`_ for more
information on that topic.
.. _Google discussion: https://github.com/searxng/searxng/issues/6359
"""
import random
import re
import string
import time
import typing as t
from urllib.parse import unquote, urlencode
@@ -47,16 +44,16 @@ about = {
"official_api_documentation": "https://developers.google.com/custom-search/",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["general", "web"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
@@ -67,23 +64,38 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
# Filter results. 0: None, 1: Moderate, 2: Strict
filter_mapping = {0: "off", 1: "medium", 2: "high"}
# https://github.com/searxng/searxng/issues/6359
nokia_useragents = (
"Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
"Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
"NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
)
# specific xpath variables
# ------------------------
# Suggestions are links placed in a *card-section*, we extract only the text
# from the links not the links itself.
suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
_arcid_range = string.ascii_letters + string.digits + "_-"
_arcid_random: tuple[str, int] | None = None
def ui_async(start: int) -> str:
"""Format of the response from UI's async request.
- ``arc_id:<...>,use_ac:true,_fmt:prog``
The arc_id is random generated every hour.
"""
global _arcid_random # pylint: disable=global-statement
use_ac = "use_ac:true"
# _fmt:html returns a HTTP 500 when user search for celebrities like
# '!google natasha allegri' or '!google chris evans'
_fmt = "_fmt:prog"
# create a new random arc_id every hour
if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
_arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
return ",".join([arc_id, use_ac, _fmt])
def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
@@ -115,11 +127,19 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
A instance of :py:obj:`babel.core.Locale` build from the
``searxng_locale`` value.
subdomain:
Google subdomain :py:obj:`google_domains` that fits to the country
code.
params:
Py-Dictionary with additional request arguments (can be passed to
:py:func:`urllib.parse.urlencode`).
- ``hl`` parameter: specifies the interface language of user interface.
- ``lr`` parameter: restricts search results to documents written in
a particular language.
- ``cr`` parameter: restricts search results to documents
originating in a particular country.
- ``ie`` parameter: sets the character encoding scheme that should
be used to interpret the query string ('utf8').
- ``oe`` parameter: sets the character encoding scheme that should
@@ -136,6 +156,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val: dict[str, t.Any] = {
"language": None,
"country": None,
"subdomain": None,
"params": {},
"headers": {},
"cookies": {},
@@ -148,7 +169,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
except babel.core.UnknownLocaleError:
locale = None
eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
lang_code = eng_lang.split("_")[-1] # lang_zh-TW --> zh-TW / lang_en --> en
country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
@@ -163,6 +184,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
ret_val["language"] = eng_lang
ret_val["country"] = country
ret_val["locale"] = locale
ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
# hl parameter:
# The hl parameter specifies the interface language (host language) of
@@ -201,11 +223,9 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
# specify a region (country) only if a region is given in the selected
# locale --> https://github.com/searxng/searxng/issues/2672
if country is not None:
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
ret_val["params"]["cr"] = ""
if len(sxng_locale.split("-")) > 1:
ret_val["params"]["cr"] = "country" + country
# gl parameter: (mandatory by Google News)
# The gl parameter value is a two-letter country code. For WebSearch
@@ -280,77 +300,88 @@ def detect_google_sorry(resp: "SXNG_Response"):
raise SearxEngineCaptchaException()
def unwrap_google_url(raw_url: str) -> str:
# remove redirector from url
if raw_url.startswith("/url?q="):
return unquote(raw_url[7:].split("&sa=U")[0])
return raw_url
def wml_dom(resp: "SXNG_Response"):
detect_google_sorry(resp)
text = resp.text
if text.lstrip().startswith("<?xml"):
text = text.split("?>", 1)[-1]
return html.fromstring(text)
def google_request(
query: str,
params: "OnlineParams",
extra_args: dict[str, t.Any] | None = None,
*,
eng_traits: EngineTraits | None = None,
use_time_range: bool = True,
use_safesearch: bool = True,
safesearch_map: dict[int, str] | None = None,
use_locales: bool = True,
) -> None:
google_info = get_google_info(params, eng_traits or traits)
if not use_locales:
google_info["params"].pop("lr")
google_info["params"].pop("cr")
start = (params["pageno"] - 1) * 10
args: dict[str, t.Any] = {
"q": query,
"sca_esv": "1",
**google_info["params"],
**(extra_args or {}),
}
if start:
args["start"] = start
if use_time_range and params["time_range"] in time_range_dict:
args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
if use_safesearch and params["safesearch"]:
args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
params["headers"]["User-Agent"] = random.choice(nokia_useragents)
def request(query: str, params: "OnlineParams") -> None:
google_request(query, params)
"""Google search request"""
# pylint: disable=line-too-long
start = (params["pageno"] - 1) * 10
google_info = get_google_info(params, traits)
# https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search"
+ "?"
+ urlencode(
{
"q": query,
**google_info["params"],
"filter": "0",
"start": start,
# 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
# 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
# 'cs' : 1,
# 'sa': 'N',
# 'yv': 3,
# 'prmd': 'vin',
# 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
# 'sa': 'N',
# 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
# formally known as use_mobile_ui
# "asearch": "arc",
# "async": str_async,
}
)
)
if params["time_range"] in time_range_dict:
query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
if params["safesearch"]:
query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
# regex match to get image map that is found inside the returned javascript:
# (function(){var s='...';var i=['...'] ...}
RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
def parse_url_images(text: str):
data_image_map = {}
for image_url, img_id in RE_DATA_IMAGE.findall(text):
data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def response(resp: "SXNG_Response"):
"""Get response from google's search request"""
# pylint: disable=too-many-branches, too-many-statements
detect_google_sorry(resp)
data_image_map = parse_url_images(resp.text)
results = EngineResults()
dom = wml_dom(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
# parse results
for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
# pylint: disable=too-many-nested-blocks
try:
title_tag = eval_xpath_getindex(
result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
)
title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
if title_tag is None:
# this not one of the common google results *section*
logger.debug("ignoring item from the result_xpath list: missing title")
continue
title = extract_text(title_tag)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
raw_url = result.get("href")
if raw_url is None:
logger.debug(
'ignoring item from the result_xpath list: missing url of title "%s"',
@@ -358,19 +389,30 @@ def response(resp: "SXNG_Response") -> EngineResults:
)
continue
url = unwrap_google_url(raw_url)
content = extract_text(
eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
)
thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
results.add(
results.types.MainResult(
url=url,
title=title or "",
content=content or "",
thumbnail=thumbnail or "",
)
)
if raw_url.startswith('/url?q='):
url = unquote(raw_url[7:].split("&sa=U")[0]) # remove the google redirector
else:
url = raw_url
content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
for item in content_nodes:
for script in item.xpath(".//script"):
script.getparent().remove(script)
content = extract_text(content_nodes[0])
# Images that are NOT the favicon
xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
thumbnail = None
if xpath_image is not None:
thumbnail = xpath_image.get("src")
if thumbnail.startswith("data:image"):
img_id = xpath_image.get("id")
if img_id:
thumbnail = data_image_map.get(img_id)
results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
except Exception as e: # pylint: disable=broad-except
logger.error(e, exc_info=True)
@@ -378,8 +420,10 @@ def response(resp: "SXNG_Response") -> EngineResults:
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
# append suggestion
results.append({"suggestion": extract_text(suggestion)})
# return results
return results
@@ -412,12 +456,14 @@ skip_countries = [
]
def fetch_traits(engine_traits: EngineTraits):
def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
"""Fetch languages from Google."""
# pylint: disable=import-outside-toplevel, too-many-branches
from searx.network import get # see https://github.com/searxng/searxng/issues/762
engine_traits.custom["supported_domains"] = {}
resp = get("https://www.google.com/preferences", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google preferences is not OK.")
@@ -468,3 +514,22 @@ def fetch_traits(engine_traits: EngineTraits):
# alias regions
engine_traits.regions["zh-CN"] = "HK"
# supported domains
if add_domains:
resp = get("https://www.google.com/supported_domains", timeout=5)
if not resp.ok:
raise RuntimeError("Response from Google supported domains is not OK.")
for domain in resp.text.split():
domain = domain.strip()
if not domain or domain in [
".google.com",
]:
continue
region = domain.split(".")[-1].upper()
engine_traits.custom["supported_domains"][region] = "www" + domain
if region == "HK":
# There is no google.cn, we use .com.hk for zh-CN
engine_traits.custom["supported_domains"]["CN"] = "www" + domain

View File

@@ -95,11 +95,12 @@ def request(query: str, params: "OnlineParams") -> None:
token = _cse_token()
google_info = get_google_info(params, traits)
info: dict[str, str] = google_info["params"]
args = {
"rsz": "filtered_cse",
"num": str(page_size),
"hl": google_info["params"]["hl"],
"hl": info["hl"],
"cselibv": token["cselibv"],
"cx": CX,
"q": query,
@@ -113,6 +114,10 @@ def request(query: str, params: "OnlineParams") -> None:
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
args["sort"] = f"date:r:{start_date}:{end_date}"
if info.get("lr"):
args["lr"] = info["lr"]
if info.get("cr"):
args["cr"] = info["cr"]
if google_info["country"] not in (None, "ZZ"):
args["gl"] = google_info["country"]
if token["exp"]:

View File

@@ -1,75 +1,122 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Images: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Images engine using the internal
Google API used by the Google Go Android app.
import typing as t
from urllib.parse import parse_qs, unquote, urlparse
This internal API offer results in
- JSON (``_fmt:json``)
- Protobuf_ (``_fmt:pb``)
- Protobuf_ compressed? (``_fmt:pc``)
- HTML (``_fmt:html``)
- Protobuf_ encoded in JSON (``_fmt:jspb``).
.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
"""
from urllib.parse import urlencode
from json import loads
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, wml_dom
from searx.result_types import EngineResults
from searx.utils import eval_xpath_list
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import (
get_google_info,
time_range_dict,
detect_google_sorry,
)
# about
about = {
"website": "https://images.google.com",
"wikidata_id": "Q521550",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://images.google.com',
"wikidata_id": 'Q521550',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'JSON',
}
# engine dependent config
categories = ["images", "web"]
categories = ['images', 'web']
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
"""`Google max 50 pages`_
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
"""
time_range_support = True
language_support = True
safesearch = True
filter_mapping = {0: "images", 1: "active", 2: "active"}
filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "isch"},
eng_traits=traits,
safesearch_map=filter_mapping,
use_locales=False,
def request(query, params):
"""Google-Image search request"""
google_info = get_google_info(params, traits)
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ '?'
+ urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
# don't urlencode this because wildly different AND bad results
# pagination uses Zero-based numbering
+ f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if params['safesearch']:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
# this ua will allow getting ~50 results instead of 10. #1641
params['headers']['User-Agent'] = (
'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
)
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
dom = wml_dom(resp)
return params
for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
qs = parse_qs(urlparse(link.get("href", "")).query)
img_src = qs.get("imgurl", [""])[0]
url = qs.get("imgrefurl", [""])[0]
if not img_src or not url:
continue
width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
tbnid = qs.get("tbnid", [""])[0]
results.add(
results.types.Image(
url=url,
title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
img_src=img_src,
thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
resolution=f"{width} x {height}" if width and height else "",
)
)
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
json_start = resp.text.find('{"ischj":')
json_data = loads(resp.text[json_start:])
for item in json_data["ischj"].get("metadata", []):
result_item = {
'url': item["result"]["referrer_url"],
'title': item["result"]["page_title"],
'content': item["text_in_grid"]["snippet"],
'source': item["result"]["site_title"],
'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
'img_src': item["original_image"]["url"],
'thumbnail_src': item["thumbnail"]["url"],
'template': 'images.html',
}
author = item["result"].get('iptc', {}).get('creator')
if author:
result_item['author'] = ', '.join(author)
copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
if copyright_notice:
result_item['source'] += ' | ' + copyright_notice
freshness_date = item["result"].get("freshness_date")
if freshness_date:
result_item['source'] += ' | ' + freshness_date
file_size = item.get('gsa', {}).get('file_size')
if file_size:
result_item['source'] += ' (%s)' % file_size
results.append(result_item)
return results

View File

@@ -1,91 +1,325 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google News: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google News engine.
Google News has a different region handling compared to Google WEB.
- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
- the hl_ argument has to be set correctly (and different to Google WEB)
- the gl_ argument is mandatory
If one of this argument is not set correctly, the request is redirected to
CONSENT dialog::
https://consent.google.com/m?continue=
The google news API ignores some parameters from the common :ref:`google API`:
- num_ : the number of search results is ignored / there is no paging all
results for a query term are in the first response.
- save_ : is ignored / Google-News results are always *SafeSearch*
.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
"""
import typing as t
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
import json
import base64
from urllib.parse import urlencode
from lxml import html
import babel
from searx import locales
from searx.utils import (
eval_xpath_getindex,
eval_xpath,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
)
from searx.engines.google import fetch_traits as _fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
detect_google_sorry,
)
from searx.enginelib.traits import EngineTraits
from searx.result_types import EngineResults
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
# about
about = {
"website": "https://www.google.com",
"website": "https://news.google.com",
"wikidata_id": "Q12020",
"official_api_documentation": "https://developers.google.com/custom-search",
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": "HTML",
}
# engine dependent config
categories = ["news"]
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
paging = False
time_range_support = False
language_support = True
safesearch = False
# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
# False here.
#
# safesearch : results are identical for safesearch=0 and safesearch=2
safesearch = True
base_url: str = "https://news.google.com"
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "nws"},
eng_traits=traits,
use_time_range=False,
use_safesearch=False,
use_locales=False,
"""Google-News search request"""
sxng_locale = params.get("searxng_locale", "en-US")
ceid: str = locales.get_engine_locale(
sxng_locale, traits.custom["ceid"], default="US:en"
) # pyright: ignore[reportAssignmentType]
google_info = get_google_info(params, traits)
google_info["subdomain"] = "news.google.com" # google news has only one domain
ceid_region, ceid_lang = ceid.split(":")
ceid_lang, ceid_suffix = (
ceid_lang.split(":")
+ [
"",
]
)[:2]
google_info["params"]["hl"] = ceid_lang
if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
if ceid_region.lower() == ceid_lang:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
elif ceid_region.lower() != ceid_lang:
if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
google_info["params"]["hl"] = ceid_lang
else:
google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
google_info["params"]["gl"] = ceid_region
query_url = (
"https://"
+ google_info["subdomain"]
+ "/search?"
+ urlencode(
{"q": query, **google_info["params"]},
)
# ceid includes a ':' character which must not be urlencoded
+ ("&ceid=%s" % ceid)
)
def _span_text(link, css_class: str):
return extract_text(
eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
allow_none=True,
)
params["url"] = query_url
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
seen = set()
for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
href = link.get("href")
if not href:
"""Get response from google's search request"""
res = EngineResults()
detect_google_sorry(resp)
# convert the text to dom
dom = html.fromstring(resp.text)
for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
if not url:
continue
if url.startswith("./"):
url = base_url + url[1:]
# The real URL is often encoded in the "jslog" attribute
jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
# Try to extract the real URL from jslog
real_url: str | None = None
if jslog:
# jslog format is usually: "95014; 5:<base64>; track:click,vis". We
# want the second part (index 1) after splitting by ";"
parts: list[str] = jslog.split(";")
if len(parts) > 1:
b64_data: str = parts[1].split(":")[-1].strip()
# Pad base64 if necessary
b64_data += "=" * (-len(b64_data) % 4)
decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
# The URL is typically the last element in the decoded array
if (
isinstance(decoded_data, list)
and isinstance(decoded_data[-1], str)
and decoded_data[-1].startswith("http")
):
real_url = decoded_data[-1]
if real_url:
url = real_url
else:
logger.error(f"no real-url found: {url}")
continue
url = unwrap_google_url(href)
if url in seen or "google.com/search" in url:
continue
title = extract_text(eval_xpath(result, "./h4")) or ""
title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
if not title:
continue
# The pub_date is mostly a string like 'yesterday', not a real timezone
# date or time. Therefore we can't use publishedDate and place the
# *pub* sting into the content.
source = _span_text(link, "dXDvrc")
pub_date = _span_text(link, "YVIcad")
thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
pub_date = extract_text(eval_xpath(result, ".//time"))
pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
content = " / ".join([x for x in [pub_origin, pub_date] if x])
seen.add(url)
results.add(
results.types.MainResult(
thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
if thumbnail and thumbnail.startswith("/"):
thumbnail = base_url + thumbnail
res.add(
res.types.MainResult(
url=url,
title=title,
content=" / ".join(x for x in [source, pub_date] if x),
thumbnail=thumbnail or "",
content=content,
thumbnail=thumbnail,
)
)
return results
return res
ceid_list = [
"AE:ar",
"AR:es-419",
"AT:de",
"AU:en",
"BD:bn",
"BE:fr",
"BE:nl",
"BG:bg",
"BR:pt-419",
"BW:en",
"CA:en",
"CA:fr",
"CH:de",
"CH:fr",
"CL:es-419",
"CN:zh-Hans",
"CO:es-419",
"CU:es-419",
"CZ:cs",
"DE:de",
"EE:et",
"EG:ar",
"ES:ca",
"ES:es",
"ET:en",
"FI:fi",
"FR:fr",
"GB:en",
"GH:en",
"GR:el",
"HK:zh-Hant",
"HU:hu",
"ID:en",
"ID:id",
"IE:en",
"IL:en",
"IL:he",
"IN:bn",
"IN:en",
"IN:gu",
"IN:hi",
"IN:ml",
"IN:mr",
"IN:pa",
"IN:ta",
"IN:te",
"IT:it",
"JP:ja",
"KE:en",
"KR:ko",
"LB:ar",
"LT:lt",
"LV:en",
"LV:lv",
"MA:fr",
"MY:en",
"MY:ms",
"NA:en",
"NG:en",
"NL:nl",
"NO:no",
"NZ:en",
"PH:en",
"PK:en",
"PL:pl",
"RO:ro",
"RS:sr",
"RU:ru",
"SA:ar",
"SE:sv",
"SG:en",
"SI:sl",
"SK:sk",
"SN:fr",
"TH:th",
"TR:tr",
"TZ:en",
"UA:ru",
"UA:uk",
"UG:en",
"US:en",
"VN:vi",
"ZA:en",
"ZW:en",
]
"""List of region/language combinations supported by Google News. Values of the
``ceid`` argument of the Google News REST API."""
_skip_values = [
"ET:en", # english (ethiopia)
"ID:en", # english (indonesia)
"LV:en", # english (latvia)
]
_ceid_locale_map = {"NO:no": "nb-NO"}
def fetch_traits(engine_traits: EngineTraits):
_fetch_traits(engine_traits, add_domains=False)
engine_traits.custom["ceid"] = {}
for ceid in ceid_list:
if ceid in _skip_values:
continue
region, lang = ceid.split(":")
x = lang.split("-")
if len(x) > 1:
if x[1] not in ["Hant", "Hans"]:
lang = x[0]
sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
try:
locale = babel.Locale.parse(sxng_locale, sep="-")
except babel.UnknownLocaleError:
print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
continue
engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid

View File

@@ -77,6 +77,8 @@ def request(query: str, params: "OnlineParams") -> None:
"""Google-Scholar search request"""
google_info = get_google_info(params, traits)
# subdomain is: scholar.google.xy
google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
args = {
"q": query,
@@ -87,7 +89,7 @@ def request(query: str, params: "OnlineParams") -> None:
}
args.update(time_range_args(params))
params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
params["cookies"] = google_info["cookies"]
params["headers"].update(google_info["headers"])

View File

@@ -1,87 +1,186 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Videos: see :py:obj:`searx.engines.google`."""
"""This is the implementation of the Google Videos engine.
import typing as t
.. admonition:: Content-Security-Policy (CSP)
This engine needs to allow images from the `data URLs`_ (prefixed with the
``data:`` scheme)::
Header set Content-Security-Policy "img-src 'self' data: ;"
.. _data URLs:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
"""
import re
from urllib.parse import urlencode, urlparse, parse_qs, unquote
from lxml import html
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import google_request, unwrap_google_url, wml_dom
from searx.result_types import EngineResults
from searx.utils import (
eval_xpath_getindex,
eval_xpath_list,
eval_xpath_getindex,
extract_text,
get_embeded_stream_url,
parse_duration_string,
)
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
from searx.search.processors import OnlineParams
from searx.engines.google import fetch_traits # pylint: disable=unused-import
from searx.engines.google import (
get_google_info,
time_range_dict,
filter_mapping,
suggestion_xpath,
detect_google_sorry,
ui_async,
)
from searx.utils import get_embeded_stream_url
# about
about = {
"website": "https://www.google.com",
"wikidata_id": "Q219885",
"official_api_documentation": "https://developers.google.com/custom-search",
"website": 'https://www.google.com',
"wikidata_id": 'Q219885',
"official_api_documentation": 'https://developers.google.com/custom-search',
"use_official_api": False,
"require_api_key": False,
"results": "XML",
"results": 'HTML',
}
# engine dependent config
categories = ["videos", "web"]
categories = ['videos', 'web']
paging = True
max_page = 50
"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
"""
language_support = True
time_range_support = True
safesearch = True
def request(query: str, params: "OnlineParams") -> None:
google_request(
query,
params,
{"tbm": "vid"},
eng_traits=traits,
use_locales=False,
# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
def parse_data_images(text: str):
data_image_map = {}
for img_id, data_image in RE_DATA_IMAGE.findall(text):
end_pos = data_image.rfind("=")
if end_pos > 0:
data_image = data_image[: end_pos + 1]
data_image_map[img_id] = data_image
logger.debug("data:image objects --> %s", list(data_image_map.keys()))
return data_image_map
def request(query, params):
"""Google-Video search request"""
google_info = get_google_info(params, traits)
start = (params['pageno'] - 1) * 10
query_url = (
'https://'
+ google_info['subdomain']
+ '/search'
+ "?"
+ urlencode(
{
'q': query,
'tbm': "vid",
'start': start,
**google_info['params'],
'asearch': 'arc',
'async': ui_async(start),
}
)
)
if params['time_range'] in time_range_dict:
query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
if 'safesearch' in params:
query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
params['url'] = query_url
def response(resp: "SXNG_Response") -> EngineResults:
results = EngineResults()
params['cookies'] = google_info['cookies']
params['headers'].update(google_info['headers'])
return params
for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
def response(resp):
"""Get response from google's search request"""
results = []
detect_google_sorry(resp)
data_image_map = parse_data_images(resp.text)
# convert the text to dom
dom = html.fromstring(resp.text)
result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
# parse results
for result in result_divs:
title = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
allow_none=True,
)
raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
if not title or not raw_url:
continue
url = unwrap_google_url(raw_url)
thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
if "/default.jpg" in thumbnail:
thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
length = None
for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
length = parse_duration_string(extract_text(span) or "")
if length:
break
results.add(
results.types.MainResult(
url=url,
title=title,
thumbnail=thumbnail,
length=length,
iframe_src=get_embeded_stream_url(url) or "",
template="videos.html",
)
url = eval_xpath_getindex(
result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
)
if url and url.startswith('/url?q='):
url = unquote(url[7:].split('&sa=U')[0])
content = extract_text(
eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
)
pub_info = extract_text(
eval_xpath_getindex(
result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
),
allow_none=True,
)
# Broader XPath to find any <img> element
thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
duration = extract_text(
eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
)
video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
# Fallback for video_id from URL if not found via XPath
if not video_id and url and 'youtube.com' in url:
parsed_url = urlparse(url)
video_id = parse_qs(parsed_url.query).get('v', [None])[0]
# Handle thumbnail
if thumbnail and thumbnail.startswith('data:image'):
img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
if img_id and img_id in data_image_map:
thumbnail = data_image_map[img_id]
else:
thumbnail = None
if not thumbnail and video_id:
thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
# Handle video embed URL
embed_url = None
if video_id:
embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
elif url:
embed_url = get_embeded_stream_url(url)
# Only append results with valid title and url
if title and url:
results.append(
{
'url': url,
'title': title,
'content': content or '',
'author': pub_info,
'thumbnail': thumbnail,
'length': duration,
'iframe_src': embed_url,
'template': 'videos.html',
}
)
# parse suggestion
for suggestion in eval_xpath_list(dom, suggestion_xpath):
results.append({'suggestion': extract_text(suggestion)})
return results

View File

@@ -10,8 +10,7 @@ from flask_babel import gettext
from searx.data import OSM_KEYS_TAGS, CURRENCIES
from searx.external_urls import get_external_url
from searx.wikidata import send_wikidata_query
from searx.engines.wikidata import sparql_string_escape, get_thumbnail
from searx.engines.wikidata import send_wikidata_query, sparql_string_escape, get_thumbnail
from searx.result_types import EngineResults
# about
@@ -291,8 +290,7 @@ def get_title_address(result):
'house_number': address_raw.get('house_number'),
'road': address_raw.get('road'),
'locality': address_raw.get(
'city',
address_raw.get('town', address_raw.get('village')), # noqa
'city', address_raw.get('town', address_raw.get('village')) # noqa
), # noqa
'postcode': address_raw.get('postcode'),
'country': address_raw.get('country'),

View File

@@ -8,29 +8,24 @@ Some implementations are shared from :ref:`wikipedia engine`.
import typing as t
import os
from hashlib import md5
from urllib.parse import urlencode, unquote
from json import loads
from dateutil.parser import isoparse
from babel.dates import format_datetime, format_date, format_time, get_datetime_format
from searx.enginelib import EngineCache
from searx.data import WIKIDATA_UNITS
from searx.network import post, get
from searx.utils import get_string_replaces_function
from searx.external_urls import area_to_osm_zoom
from searx.utils import searxng_useragent, get_string_replaces_function
from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
from searx.engines.wikipedia import (
fetch_wikimedia_traits,
get_wiki_params,
)
from searx.enginelib.traits import EngineTraits
from searx.wikidata_properties import (
QUERY_TEMPLATE,
WDArticle,
WDAttrList,
WDGeoAttribute,
WDImageAttribute,
WDURLAttribute,
get_attributes,
)
from searx.wikidata import SPARQL_ENDPOINT_URL, SPARQL_EXPLAIN_URL, get_wikidata_headers
if t.TYPE_CHECKING:
from searx.extended_types import SXNG_Response
@@ -53,6 +48,78 @@ display_type = ["infobox"]
one will add a hit to the result list. The first one will show a hit in the
info box. Both values can be set, or one of the two can be set."""
CACHE: EngineCache
"""Persistent (SQLite) key/value cache that deletes its values after ``expire``
seconds."""
# SPARQL
SPARQL_ENDPOINT_URL = "https://query.wikidata.org/sparql"
SPARQL_EXPLAIN_URL = "https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain"
WDPType = dict[str | tuple[str, str], str]
WIKIDATA_PROPERTIES: WDPType = {
"P434": "MusicBrainz",
"P435": "MusicBrainz",
"P436": "MusicBrainz",
"P966": "MusicBrainz",
"P345": "IMDb",
"P2397": "YouTube",
"P1651": "YouTube",
"P2002": "Twitter",
"P2013": "Facebook",
"P2003": "Instagram",
"P4033": "Mastodon",
"P11947": "Lemmy",
"P12622": "PeerTube",
}
# SERVICE wikibase:mwapi : https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual/MWAPI
# SERVICE wikibase:label: https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_Label#Manual_Label_SERVICE
# https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates
# https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format#Data_model
# optimization:
# * https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/query_optimization
# * https://github.com/blazegraph/database/wiki/QueryHints
QUERY_TEMPLATE = """
SELECT ?item ?itemLabel ?itemDescription ?lat ?long %SELECT%
WHERE
{
SERVICE wikibase:mwapi {
bd:serviceParam wikibase:endpoint "www.wikidata.org";
wikibase:api "EntitySearch";
wikibase:limit 1;
mwapi:search "%QUERY%";
mwapi:language "%LANGUAGE%".
?item wikibase:apiOutputItem mwapi:item.
}
hint:Prior hint:runFirst "true".
%WHERE%
SERVICE wikibase:label {
bd:serviceParam wikibase:language "%LANGUAGE%,en".
?item rdfs:label ?itemLabel .
?item schema:description ?itemDescription .
%WIKIBASE_LABELS%
}
}
GROUP BY ?item ?itemLabel ?itemDescription ?lat ?long %GROUP_BY%
"""
# Get the calendar names and the property names
QUERY_PROPERTY_NAMES = """
SELECT ?item ?name
WHERE {
{
SELECT ?item
WHERE { ?item wdt:P279* wd:Q12132 }
} UNION {
VALUES ?item { %ATTRIBUTES% }
}
OPTIONAL { ?item rdfs:label ?name. }
}
"""
# see the property "dummy value" of https://www.wikidata.org/wiki/Q2013 (Wikidata)
# hard coded here to avoid to an additional SPARQL request when the server starts
DUMMY_ENTITY_URLS = set(
@@ -64,13 +131,349 @@ DUMMY_ENTITY_URLS = set(
# https://lists.w3.org/Archives/Public/public-rdf-dawg/2011OctDec/0175.html
sparql_string_escape = get_string_replaces_function(
# fmt: off
{"\t": "\\\t", "\n": "\\\n", "\r": "\\\r", "\b": "\\\b", "\f": "\\\f", "\"": "\\\"", "'": "\\'", "\\": "\\\\"}
{
"\t": "\\\t",
"\n": "\\\n",
"\r": "\\\r",
"\b": "\\\b",
"\f": "\\\f",
"\"": "\\\"",
"\'": "\\\'",
"\\": "\\\\"
}
# fmt: on
)
replace_http_by_https = get_string_replaces_function({"http:": "https:"})
class WDAttribute:
def __init__(self, name: str):
self.name: str = name
def get_select(self):
return "(group_concat(distinct ?{name};separator=', ') as ?{name}s)".replace("{name}", self.name)
def get_label(self, language: str):
return get_label_for_entity(self.name, language)
def get_where(self):
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace("{name}", self.name)
def get_wikibase_label(self) -> str:
return ""
def get_group_by(self) -> str:
return ""
def get_str(self, result: dict[str, t.Any], language: str) -> str | None: # pylint: disable=unused-argument
return result.get(self.name + "s")
def __repr__(self):
return "<" + str(type(self).__name__) + ":" + self.name + ">"
class WDAmountAttribute(WDAttribute):
def get_select(self) -> str:
return "?{name} ?{name}Unit".replace("{name}", self.name)
def get_where(self):
return """ OPTIONAL { ?item p:{name} ?{name}Node .
?{name}Node rdf:type wikibase:BestRank ; ps:{name} ?{name} .
OPTIONAL { ?{name}Node psv:{name}/wikibase:quantityUnit ?{name}Unit. } }""".replace('{name}', self.name)
def get_group_by(self) -> str:
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name)
unit: str | None = result.get(self.name + "Unit")
if unit is not None:
unit = unit.replace("http://www.wikidata.org/entity/", "")
return str(value) + " " + get_label_for_entity(unit, language)
return value
class WDArticle(WDAttribute):
def __init__(self, language: str, kwargs: dict[str, t.Any] | None = None):
super().__init__("wikipedia")
self.language: str = language
self.kwargs: dict[str, t.Any] = kwargs or {}
def get_label(self, language: str):
# language parameter is ignored
return "Wikipedia ({language})".replace("{language}", self.language)
def get_select(self):
return "?article{language} ?articleName{language}".replace("{language}", self.language)
def get_where(self):
return """OPTIONAL { ?article{language} schema:about ?item ;
schema:inLanguage "{language}" ;
schema:isPartOf <https://{language}.wikipedia.org/> ;
schema:name ?articleName{language} . }""".replace('{language}', self.language)
def get_group_by(self):
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
key = "article{language}".replace("{language}", self.language)
return result.get(key)
class WDLabelAttribute(WDAttribute):
def get_select(self):
return "(group_concat(distinct ?{name}Label;separator=', ') as ?{name}Labels)".replace("{name}", self.name)
def get_where(self):
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace("{name}", self.name)
def get_wikibase_label(self) -> str:
return "?{name} rdfs:label ?{name}Label .".replace("{name}", self.name)
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
return result.get(self.name + "Labels")
class WDURLAttribute(WDAttribute):
HTTP_WIKIMEDIA_IMAGE: str = "http://commons.wikimedia.org/wiki/Special:FilePath/"
def __init__(
self,
name: str,
url_id: str | None = None,
url_path_prefix: str | None = None,
kwargs: dict[str, t.Any] | None = None,
):
"""
:param url_id: ID matching one key in ``external_urls.json`` for
converting IDs to full URLs.
:param url_path_prefix: Path prefix if the values are of format
``account@domain``. If provided, value are rewritten to
``https://<domain><url_path_prefix><account>``. For example::
WDURLAttribute('P4033', url_path_prefix='/@')
Adds Property `P4033 <https://www.wikidata.org/wiki/Property:P4033>`_
to the wikidata query. This field might return for example
``libreoffice@fosstodon.org`` and the URL built from this is then:
- account: ``libreoffice``
- domain: ``fosstodon.org``
- result url: https://fosstodon.org/@libreoffice
"""
super().__init__(name)
self.url_id: str | None = url_id
self.url_path_prefix: str | None = url_path_prefix
self.kwargs: dict[str, t.Any] = kwargs or {}
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name + "s")
if not value:
return None
value = value.split(",")[0]
if self.url_id:
url_id = self.url_id
if value.startswith(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE):
value = value[len(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE) :]
url_id = "wikimedia_image"
return get_external_url(url_id, value)
if self.url_path_prefix:
[account, domain] = [x.strip("@ ") for x in value.rsplit("@", 1)]
return f"https://{domain}{self.url_path_prefix}{account}"
return value
class WDGeoAttribute(WDAttribute):
def get_label(self, language: str):
return "OpenStreetMap"
def get_select(self):
return "?{name}Lat ?{name}Long".replace("{name}", self.name)
def get_where(self):
return """OPTIONAL { ?item p:{name}/psv:{name} [
wikibase:geoLatitude ?{name}Lat ;
wikibase:geoLongitude ?{name}Long ] }""".replace('{name}', self.name)
def get_group_by(self):
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
latitude: str | None = result.get(self.name + "Lat")
longitude: str | None = result.get(self.name + "Long")
if latitude and longitude:
return latitude + " " + longitude
return None
def get_geo_url(self, result: dict[str, t.Any], osm_zoom: int = 19) -> str | None:
latitude: str | None = result.get(self.name + "Lat")
longitude: str | None = result.get(self.name + "Long")
if latitude and longitude:
return get_earth_coordinates_url(latitude, longitude, osm_zoom)
return None
class WDImageAttribute(WDURLAttribute):
def __init__(self, name: str, url_id: str | None = None, priority: int = 100):
super().__init__(name, url_id)
self.priority: int = priority
class WDDateAttribute(WDAttribute):
def get_select(self):
return "?{name} ?{name}timePrecision ?{name}timeZone ?{name}timeCalendar".replace("{name}", self.name)
def get_where(self):
# To remove duplicate, add
# FILTER NOT EXISTS { ?item p:{name}/psv:{name}/wikibase:timeValue ?{name}bis FILTER (?{name}bis < ?{name}) }
# this filter is too slow, so the response function ignore duplicate results
# (see the seen_entities variable)
return """OPTIONAL { ?item p:{name}/psv:{name} [
wikibase:timeValue ?{name} ;
wikibase:timePrecision ?{name}timePrecision ;
wikibase:timeTimezone ?{name}timeZone ;
wikibase:timeCalendarModel ?{name}timeCalendar ] . }
hint:Prior hint:rangeSafe true;""".replace('{name}', self.name)
def get_group_by(self):
return self.get_select()
def format_8(self, value: str, locale: str) -> str: # pylint: disable=unused-argument
# precision: less than a year
return value
def format_9(self, value: str, locale: str) -> str:
year = int(value)
# precision: year
if year < 1584:
if year < 0:
return str(year - 1)
return str(year)
timestamp = isoparse(value)
return format_date(timestamp, format="yyyy", locale=locale)
def format_10(self, value: str, locale: str) -> str:
# precision: month
timestamp = isoparse(value)
return format_date(timestamp, format="MMMM y", locale=locale)
def format_11(self, value: str, locale: str) -> str:
# precision: day
timestamp = isoparse(value)
return format_date(timestamp, format="full", locale=locale)
def format_13(self, value: str, locale: str) -> str:
timestamp = isoparse(value)
# precision: minute
return (
get_datetime_format(format, locale=locale)
.replace("'", "")
.replace("{0}", format_time(timestamp, "full", tzinfo=None, locale=locale))
.replace("{1}", format_date(timestamp, "short", locale=locale))
)
def format_14(self, value: str, locale: str) -> str:
# precision: second.
return format_datetime(isoparse(value), format="full", locale=locale)
DATE_FORMAT: dict[str, tuple[str, int]] = {
"0": ("format_8", 1000000000),
"1": ("format_8", 100000000),
"2": ("format_8", 10000000),
"3": ("format_8", 1000000),
"4": ("format_8", 100000),
"5": ("format_8", 10000),
"6": ("format_8", 1000),
"7": ("format_8", 100),
"8": ("format_8", 10),
"9": ("format_9", 1), # year
"10": ("format_10", 1), # month
"11": ("format_11", 0), # day
"12": ("format_13", 0), # hour (not supported by babel, display minute)
"13": ("format_13", 0), # minute
"14": ("format_14", 0), # second
}
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name)
if value == "" or value is None:
return None
_p: str = result.get(self.name + "timePrecision") or "1"
date_format = WDDateAttribute.DATE_FORMAT.get(_p)
if date_format is not None:
format_method = getattr(self, date_format[0])
precision: int = date_format[1]
try:
if precision >= 1:
_t = value.split("-")
if value.startswith("-"):
value = "-" + _t[1]
else:
value = _t[0]
return format_method(value, language)
except Exception: # pylint: disable=broad-except
return value
return value
WDAttrType = (
WDAttribute
| WDAmountAttribute
| WDArticle
| WDLabelAttribute
| WDURLAttribute
| WDGeoAttribute
| WDImageAttribute
| WDDateAttribute
)
WDAttrList = list[WDAttrType]
def get_headers() -> dict[str, str]:
# user agent: https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits
return {
"Accept": "application/sparql-results+json",
"User-Agent": f"wikidata engine - {searxng_useragent()}",
}
def get_label_for_entity(entity_id: str, language: str) -> str:
name = WIKIDATA_PROPERTIES.get(entity_id)
if name is None:
name = WIKIDATA_PROPERTIES.get((entity_id, language))
if name is None:
name = WIKIDATA_PROPERTIES.get((entity_id, language.split("-")[0]))
if name is None:
name = WIKIDATA_PROPERTIES.get((entity_id, "en"))
if name is None:
name = entity_id
return name
def send_wikidata_query(query: str, method: str = "GET", **kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
if method == "GET":
# query will be cached by wikidata
http_response = get(SPARQL_ENDPOINT_URL + "?" + urlencode({"query": query}), headers=get_headers(), **kwargs)
else:
# query won't be cached by wikidata
http_response = post(SPARQL_ENDPOINT_URL, data={"query": query}, headers=get_headers(), **kwargs)
if http_response.status_code != 200:
logger.debug("SPARQL endpoint error %s", http_response.content.decode())
logger.debug("request time %s", str(http_response.elapsed))
http_response.raise_for_status()
return loads(http_response.content.decode())
def request(query: str, params: "OnlineParams") -> None:
attributes: WDAttrList
@@ -81,7 +484,7 @@ def request(query: str, params: "OnlineParams") -> None:
params["method"] = "POST"
params["url"] = SPARQL_ENDPOINT_URL
params["data"] = {"query": query}
params["headers"] = get_wikidata_headers()
params["headers"] = get_headers()
# additional parameters (not a part of OnlineParams)
params["language"] = eng_tag # type: ignore
@@ -174,6 +577,7 @@ def get_results(
for attribute in attributes:
value: str | None = attribute.get_str(attribute_result, language)
if value is not None and value != "":
if isinstance(attribute, (WDURLAttribute, WDArticle)):
# get_select() method : there is group_concat(distinct ...;separator=", ")
# split the value here
@@ -259,15 +663,212 @@ def get_query(query: str, language: str) -> tuple[str, WDAttrList]:
return query, attributes
def get_attributes(language: str):
# pylint: disable=too-many-statements
attributes: WDAttrList = []
def add_value(name: str):
attributes.append(WDAttribute(name))
def add_amount(name: str):
attributes.append(WDAmountAttribute(name))
def add_label(name: str):
attributes.append(WDLabelAttribute(name))
def add_url(name: str, url_id: str | None = None, url_path_prefix: str | None = None, **kwargs: dict[str, t.Any]):
attributes.append(WDURLAttribute(name, url_id, url_path_prefix, kwargs))
def add_image(name: str, url_id: str | None = None, priority: int = 1):
attributes.append(WDImageAttribute(name, url_id, priority))
def add_date(name: str):
attributes.append(WDDateAttribute(name))
# Dates
for p in [
"P571", # inception date
"P576", # dissolution date
"P580", # start date
"P582", # end date
"P569", # date of birth
"P570", # date of death
"P619", # date of spacecraft launch
"P620",
]: # date of spacecraft landing
add_date(p)
for p in [
"P27", # country of citizenship
"P495", # country of origin
"P17", # country
"P159",
]: # headquarters location
add_label(p)
# Places
for p in [
"P36", # capital
"P35", # head of state
"P6", # head of government
"P122", # basic form of government
"P37",
]: # official language
add_label(p)
add_value("P1082") # population
add_amount("P2046") # area
add_amount("P281") # postal code
add_label("P38") # currency
add_amount("P2048") # height (building)
# Media
for p in [
"P400", # platform (videogames, computing)
"P50", # author
"P170", # creator
"P57", # director
"P175", # performer
"P178", # developer
"P162", # producer
"P176", # manufacturer
"P58", # screenwriter
"P272", # production company
"P264", # record label
"P123", # publisher
"P449", # original network
"P750", # distributed by
"P86",
]: # composer
add_label(p)
add_date("P577") # publication date
add_label("P136") # genre (music, film, artistic...)
add_label("P364") # original language
add_value("P212") # ISBN-13
add_value("P957") # ISBN-10
add_label("P275") # copyright license
add_label("P277") # programming language
add_value("P348") # version
add_label("P840") # narrative location
# Languages
add_value("P1098") # number of speakers
add_label("P282") # writing system
add_label("P1018") # language regulatory body
add_value("P218") # language code (ISO 639-1)
# Other
add_label("P169") # ceo
add_label("P112") # founded by
add_label("P1454") # legal form (company, organization)
add_label("P137") # operator (service, facility, ...)
add_label("P1029") # crew members (tripulation)
add_label("P225") # taxon name
add_value("P274") # chemical formula
add_label("P1346") # winner (sports, contests, ...)
add_value("P1120") # number of deaths
add_value("P498") # currency code (ISO 4217)
# URL
kwargs: dict[str, t.Any] = {"official": True}
add_url("P856", **kwargs) # official website
attributes.append(WDArticle(language)) # wikipedia (user language)
if not language.startswith("en"):
attributes.append(WDArticle("en")) # wikipedia (english)
add_url("P1324") # source code repository
add_url("P1581") # blog
add_url("P434", url_id="musicbrainz_artist")
add_url("P435", url_id="musicbrainz_work")
add_url("P436", url_id="musicbrainz_release_group")
add_url("P966", url_id="musicbrainz_label")
add_url("P345", url_id="imdb_id")
add_url("P2397", url_id="youtube_channel")
add_url("P1651", url_id="youtube_video")
add_url("P2002", url_id="twitter_profile")
add_url("P2013", url_id="facebook_profile")
add_url("P2003", url_id="instagram_profile")
# Fediverse
add_url("P4033", url_path_prefix="/@") # Mastodon user
add_url("P11947", url_path_prefix="/c/") # Lemmy community
add_url("P12622", url_path_prefix="/c/") # PeerTube channel
# Map
attributes.append(WDGeoAttribute("P625"))
# Image
add_image("P15", priority=1, url_id="wikimedia_image") # route map
add_image("P242", priority=2, url_id="wikimedia_image") # locator map
add_image("P154", priority=3, url_id="wikimedia_image") # logo
add_image("P18", priority=4, url_id="wikimedia_image") # image
add_image("P41", priority=5, url_id="wikimedia_image") # flag
add_image("P2716", priority=6, url_id="wikimedia_image") # collage
add_image("P2910", priority=7, url_id="wikimedia_image") # icon
return attributes
def debug_explain_wikidata_query(query: str, method: str = "GET"):
if method == "GET":
http_response = get(SPARQL_EXPLAIN_URL + "&" + urlencode({"query": query}), headers=get_wikidata_headers())
http_response = get(SPARQL_EXPLAIN_URL + "&" + urlencode({"query": query}), headers=get_headers())
else:
http_response = post(SPARQL_EXPLAIN_URL, data={"query": query}, headers=get_wikidata_headers())
http_response = post(SPARQL_EXPLAIN_URL, data={"query": query}, headers=get_headers())
http_response.raise_for_status()
return http_response.content
def init(_):
global CACHE # pylint: disable=global-statement
CACHE = EngineCache("wikidata")
# In an environment with competing processes, the initial loading of the
# cache is required only once.
eng_state: str | None = CACHE.get("eng_state")
if not eng_state or not eng_state.startswith("STATE:"):
CACHE.set("eng_state", f"STATE: being initialized by PID {os.getpid()}")
try:
init_wikidata_properties()
except Exception:
CACHE.set("eng_state", f"ERROR: initialization by PID {os.getpid()} failed.")
raise
else:
logger.debug(eng_state)
def init_wikidata_properties():
global WIKIDATA_PROPERTIES # pylint: disable=global-statement
p: WDPType = CACHE.get(key="WIKIDATA_PROPERTIES")
if p:
WIKIDATA_PROPERTIES = p
return
# WIKIDATA_PROPERTIES : add unit symbols
for k, v in WIKIDATA_UNITS.items():
WIKIDATA_PROPERTIES[k] = v["symbol"]
# WIKIDATA_PROPERTIES : add property labels
wikidata_property_names: list[str] = []
for attribute in get_attributes("en"):
if type(attribute) in (WDAttribute, WDAmountAttribute, WDURLAttribute, WDDateAttribute, WDLabelAttribute):
if attribute.name not in WIKIDATA_PROPERTIES:
wikidata_property_names.append("wd:" + attribute.name)
query = QUERY_PROPERTY_NAMES.replace("%ATTRIBUTES%", " ".join(wikidata_property_names))
kwargs: dict[str, t.Any] = {"timeout": 20}
jsonresponse = send_wikidata_query(query, **kwargs)
for result in jsonresponse.get("results", {}).get("bindings", {}):
name_field = result.get("name")
if not name_field:
continue
name = name_field["value"]
lang = name_field["xml:lang"]
entity_id = result["item"]["value"].replace("http://www.wikidata.org/entity/", "")
WIKIDATA_PROPERTIES[(entity_id, lang)] = name.capitalize()
CACHE.set(key="WIKIDATA_PROPERTIES", value=WIKIDATA_PROPERTIES)
def fetch_traits(engine_traits: EngineTraits):
"""Uses languages evaluated from :py:obj:`wikipedia.fetch_wikimedia_traits
<searx.engines.wikipedia.fetch_wikimedia_traits>` and removes

View File

@@ -6,7 +6,7 @@
import typing as t
from base64 import urlsafe_b64encode, urlsafe_b64decode
from zlib import compress, decompressobj
from zlib import compress, decompress
from urllib.parse import parse_qs, urlencode
from collections import OrderedDict
from collections.abc import Iterable
@@ -517,7 +517,7 @@ class Preferences:
def parse_encoded_data(self, input_data: str):
"""parse (base64) preferences from request (``flask.request.form['preferences']``)"""
bin_data = decompressobj().decompress(urlsafe_b64decode(input_data), 16 * 1024)
bin_data = decompress(urlsafe_b64decode(input_data))
dict_data = {}
for x, y in parse_qs(bin_data.decode('ascii'), keep_blank_values=True).items():
dict_data[x] = y[0]

View File

@@ -1217,12 +1217,12 @@ engines:
- name: google
engine: google
shortcut: go
disabled: true
inactive: true
- name: google images
engine: google_images
shortcut: goi
disabled: true
inactive: true
- name: google news
engine: google_news
@@ -1231,6 +1231,7 @@ engines:
- name: google videos
engine: google_videos
shortcut: gov
inactive: true
- name: google cse
engine: google_cse

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -24,7 +24,7 @@ msgstr ""
"Project-Id-Version: searx\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2026-07-15 15:45+0000\n"
"PO-Revision-Date: 2026-08-21 20:54+0000\n"
"PO-Revision-Date: 2026-07-17 12:20+0000\n"
"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
"Language-Team: Bulgarian <https://translate.codeberg.org/projects/searxng/"
"searxng/bg/>\n"
@@ -33,7 +33,7 @@ msgstr ""
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=n != 1;\n"
"X-Generator: Weblate 2026.8.1\n"
"X-Generator: Weblate 2026.6.1\n"
"Generated-By: Babel 2.18.0\n"
#. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -737,7 +737,7 @@ msgstr ""
#: searx/plugins/calculator.py:25
msgid "Calculator"
msgstr "Калкулатор"
msgstr ""
#: searx/plugins/calculator.py:26
msgid "Parses and solves mathematical expressions."
@@ -1653,11 +1653,11 @@ msgstr "Резолюция"
#: searx/templates/simple/result_templates/images.html:55
msgid "Image formats"
msgstr "формат на изображението"
msgstr ""
#: searx/templates/simple/result_templates/images.html:56
msgid "original format"
msgstr "оригинален формат"
msgstr ""
#: searx/templates/simple/result_templates/images.html:64
msgid "View source"

View File

@@ -46,20 +46,19 @@
# MaiuZ <maiuz@noreply.codeberg.org>, 2025.
msgid ""
msgstr ""
"Project-Id-Version: searx\n"
"Project-Id-Version: searx\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2026-07-15 15:45+0000\n"
"PO-Revision-Date: 2026-08-21 20:54+0000\n"
"PO-Revision-Date: 2026-05-19 12:07+0000\n"
"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
"Language: it\n"
"Language-Team: Italian <https://translate.codeberg.org/projects/searxng/"
"searxng/it/>\n"
"Language-Team: Italian "
"<https://translate.codeberg.org/projects/searxng/searxng/it/>\n"
"Plural-Forms: nplurals=2; plural=n != 1;\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.18.0\n"
"X-Generator: Weblate 2026.8.1\n"
#. CONSTANT_NAMES['NO_SUBGROUPING']
#: searx/searxng.msg
@@ -1678,11 +1677,11 @@ msgstr "Risoluzione"
#: searx/templates/simple/result_templates/images.html:55
msgid "Image formats"
msgstr "Formati di immagine"
msgstr ""
#: searx/templates/simple/result_templates/images.html:56
msgid "original format"
msgstr "formato originale"
msgstr ""
#: searx/templates/simple/result_templates/images.html:64
msgid "View source"
@@ -2457,3 +2456,4 @@ msgstr "nascondi video"
#~ msgid "Engine"
#~ msgstr "Motore"

View File

@@ -14,25 +14,23 @@
# Mooo <mooo@users.noreply.translate.codeberg.org>, 2025.
# naktinis <naktinis@users.noreply.translate.codeberg.org>, 2025.
# return42 <return42@noreply.codeberg.org>, 2025, 2026.
# Mooo <mooo@noreply.codeberg.org>, 2026.
msgid ""
msgstr ""
"Project-Id-Version: searx\n"
"Project-Id-Version: searx\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2026-07-15 15:45+0000\n"
"PO-Revision-Date: 2026-08-14 19:38+0000\n"
"Last-Translator: Mooo <mooo@noreply.codeberg.org>\n"
"PO-Revision-Date: 2026-05-19 12:07+0000\n"
"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
"Language: lt\n"
"Language-Team: Lithuanian <https://translate.codeberg.org/projects/searxng/"
"searxng/lt/>\n"
"Plural-Forms: nplurals=4; plural=(n % 10 == 1 && (n % 100 > 19 || n % 100 < "
"11) ? 0 : (n % 10 >= 2 && n % 10 <=9) && (n % 100 > 19 || n % 100 < 11) ? 1 "
": n % 1 != 0 ? 2: 3);\n"
"Language-Team: Lithuanian "
"<https://translate.codeberg.org/projects/searxng/searxng/lt/>\n"
"Plural-Forms: nplurals=4; plural=(n % 10 == 1 && (n % 100 > 19 || n % 100"
" < 11) ? 0 : (n % 10 >= 2 && n % 10 <=9) && (n % 100 > 19 || n % 100 < "
"11) ? 1 : n % 1 != 0 ? 2: 3);\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.18.0\n"
"X-Generator: Weblate 2026.8.1\n"
#. CONSTANT_NAMES['NO_SUBGROUPING']
#: searx/searxng.msg
@@ -42,7 +40,7 @@ msgstr "be tolesnio pogrupio"
#. CONSTANT_NAMES['DEFAULT_CATEGORY']
#: searx/searxng.msg
msgid "other"
msgstr "kita"
msgstr "kitas"
#. CATEGORY_NAMES['FILES']
#: searx/searxng.msg
@@ -82,7 +80,7 @@ msgstr "radijas"
#. CATEGORY_NAMES['TV']
#: searx/searxng.msg
msgid "tv"
msgstr "televizija"
msgstr "televizorius"
#. CATEGORY_NAMES['IT']
#: searx/searxng.msg
@@ -132,7 +130,7 @@ msgstr "paketai"
#. CATEGORY_GROUPS['Q_A']
#: searx/searxng.msg
msgid "q&a"
msgstr "klausimai ir atsakymai"
msgstr "Dažnai užduodami klausymai"
#. CATEGORY_GROUPS['REPOS']
#: searx/searxng.msg
@@ -162,17 +160,17 @@ msgstr "automatinis"
#. STYLE_NAMES['LIGHT']
#: searx/searxng.msg
msgid "light"
msgstr "šviesus"
msgstr "šviesi"
#. STYLE_NAMES['DARK']
#: searx/searxng.msg
msgid "dark"
msgstr "tamsus"
msgstr "tamsi"
#. STYLE_NAMES['BLACK']
#: searx/searxng.msg
msgid "black"
msgstr "juodas"
msgstr "juoda"
#. BRAND_CUSTOM_LINKS['UPTIME']
#: searx/searxng.msg
@@ -187,22 +185,22 @@ msgstr "Apie"
#. WEATHER_TERMS['AVERAGE TEMP.']
#: searx/searxng.msg
msgid "Average temp."
msgstr "Vidutinė temperatūra"
msgstr "Vidutinė temperatura"
#. WEATHER_TERMS['CLOUD COVER']
#: searx/searxng.msg
msgid "Cloud cover"
msgstr "Debesų padengimas"
msgstr "Debesio serveris"
#. WEATHER_TERMS['CONDITION']
#: searx/searxng.msg
msgid "Condition"
msgstr "Sąlygos"
msgstr "Sąlyga"
#. WEATHER_TERMS['CURRENT CONDITION']
#: searx/searxng.msg
msgid "Current condition"
msgstr "Dabartinės sąlygos"
msgstr "Esamos sąlygos"
#. WEATHER_TERMS['EVENING']
#: searx/searxng.msg
@@ -222,12 +220,12 @@ msgstr "Dregmė"
#. WEATHER_TERMS['MAX TEMP.']
#: searx/searxng.msg
msgid "Max temp."
msgstr "Aukščiausia temperatūra"
msgstr "Aukščiausia temperatura"
#. WEATHER_TERMS['MIN TEMP.']
#: searx/searxng.msg
msgid "Min temp."
msgstr "Mažiausia temperatūra"
msgstr "Mažiausia temperatura"
#. WEATHER_TERMS['MORNING']
#: searx/searxng.msg
@@ -262,7 +260,7 @@ msgstr "Saulėlydis"
#. WEATHER_TERMS['TEMPERATURE']
#: searx/searxng.msg searx/templates/simple/answer/weather.html:17
msgid "Temperature"
msgstr "Temperatūra"
msgstr "Temperatura"
#. WEATHER_TERMS['UV INDEX']
#: searx/searxng.msg
@@ -282,207 +280,207 @@ msgstr "Vėjas"
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Clear sky"
msgstr "Giedra"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Partly cloudy"
msgstr "Šiek tiek debesuota"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Cloudy"
msgstr "Debesuota"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Fair"
msgstr "Nedebesuota"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Fog"
msgstr "Rūkas"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light rain and thunder"
msgstr "Silpnas lietus su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light rain showers and thunder"
msgstr "Silpnas liūtinis lietus su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light rain showers"
msgstr "Silpnas liūtinis lietus"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light rain"
msgstr "Silpnas lietus"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Rain and thunder"
msgstr "Lietus su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Rain showers and thunder"
msgstr "Liūtinis lietus su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Rain showers"
msgstr "Liūtinis lietus"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Rain"
msgstr "Lietus"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy rain and thunder"
msgstr "Stiprus lietus su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy rain showers and thunder"
msgstr "Stiprus liūtinis lietus su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy rain showers"
msgstr "Stiprus liūtinis lietus"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy rain"
msgstr "Stiprus lietus"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light sleet and thunder"
msgstr "Silpna šlapdriba su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light sleet showers and thunder"
msgstr "Silpna šlapdriba, liūtis su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light sleet showers"
msgstr "Silpna šlapdriba, liūtis"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light sleet"
msgstr "Silpna šlapdriba"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Sleet and thunder"
msgstr "Šlapdriba su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Sleet showers and thunder"
msgstr "Šlapdriba, liūtis su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Sleet showers"
msgstr "Šlapdriba, liūtis"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Sleet"
msgstr "Šlapdriba"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy sleet and thunder"
msgstr "Stipri šlapdriba su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy sleet showers and thunder"
msgstr "Stipri šlapdriba, liūtis su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy sleet showers"
msgstr "Stipri šlapdriba, liūtis"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy sleet"
msgstr "Stipri šlapdriba"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light snow and thunder"
msgstr "Silpnas sniegas su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light snow showers and thunder"
msgstr "Silpnas liūtinis sniegas su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light snow showers"
msgstr "Silpnas liūtinis sniegas"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Light snow"
msgstr "Silpnas sniegas"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Snow and thunder"
msgstr "Sniegas su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Snow showers and thunder"
msgstr "Liūtinis sniegas su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Snow showers"
msgstr "Liūtinis sniegas"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Snow"
msgstr "Sniegas"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy snow and thunder"
msgstr "Stiprus sniegas su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy snow showers and thunder"
msgstr "Stiprus liūtinis sniegas su perkūnija"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy snow showers"
msgstr "Stiprus liūtinis sniegas"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy snow"
msgstr "Stiprus sniegas"
msgstr ""
#. SOCIAL_MEDIA_TERMS['SUBSCRIBERS']
#: searx/engines/lemmy.py:85 searx/searxng.msg
@@ -497,7 +495,7 @@ msgstr "Įrašai"
#. SOCIAL_MEDIA_TERMS['ACTIVE USERS']
#: searx/engines/lemmy.py:87 searx/searxng.msg
msgid "active users"
msgstr "Aktyvūs naudotojai"
msgstr "Aktyvus naudotojai"
#. SOCIAL_MEDIA_TERMS['COMMENTS']
#: searx/engines/discourse.py:157 searx/engines/hackernews.py:83
@@ -508,12 +506,12 @@ msgstr "Komentarai"
#. SOCIAL_MEDIA_TERMS['USER']
#: searx/engines/lemmy.py:129 searx/engines/lemmy.py:164 searx/searxng.msg
msgid "user"
msgstr "Naudotojas"
msgstr "Naudotojai"
#. SOCIAL_MEDIA_TERMS['COMMUNITY']
#: searx/engines/lemmy.py:131 searx/engines/lemmy.py:165 searx/searxng.msg
msgid "community"
msgstr "bendruomenė"
msgstr "Bendruomene"
#. SOCIAL_MEDIA_TERMS['POINTS']
#: searx/engines/hackernews.py:83 searx/searxng.msg
@@ -556,11 +554,11 @@ msgstr "Šaltinis"
#: searx/webapp.py:330
msgid "Error loading the next page"
msgstr "Klaida įkeliant kitą puslapį"
msgstr "Klaida keliant kitą puslapį"
#: searx/webapp.py:478 searx/webapp.py:876
msgid "Invalid settings, please edit your preferences"
msgstr "Neteisingi nustatymai, pataisykite nuostatas"
msgstr "Neteisingi nustatymai, pakeiskite savo nuostatas"
#: searx/webapp.py:494
msgid "Invalid settings"
@@ -576,7 +574,7 @@ msgstr "laikas baigėsi"
#: searx/webutils.py:37
msgid "parsing error"
msgstr "nagrinėjimo klaida"
msgstr "parsavymo klaida"
#: searx/webutils.py:38
msgid "HTTP protocol error"
@@ -592,7 +590,7 @@ msgstr "SSL klaida: liudijimo tikrinimas patyrė nesėkmę"
#: searx/webutils.py:42
msgid "unexpected crash"
msgstr "netikėta strigtis"
msgstr "netikėta klaida"
#: searx/webutils.py:49
msgid "HTTP error"
@@ -604,11 +602,11 @@ msgstr "HTTP ryšio klaida"
#: searx/webutils.py:56
msgid "proxy error"
msgstr "įgaliotojo serverio klaida"
msgstr "persiuntimų serverio klaida"
#: searx/webutils.py:57
msgid "CAPTCHA"
msgstr "saugos kodas"
msgstr "CAPTCHA"
#: searx/webutils.py:58
msgid "too many requests"
@@ -624,31 +622,31 @@ msgstr "serverio API klaida"
#: searx/webutils.py:79
msgid "Suspended"
msgstr "Pristabdyta"
msgstr "Sustabdytas"
#: searx/webutils.py:307
#, python-brace-format
msgid "{minutes} minute(s) ago"
msgstr "prieš {minutes} min."
msgstr "prieš {minutes} min"
#: searx/webutils.py:308
#, python-brace-format
msgid "{hours} hour(s), {minutes} minute(s) ago"
msgstr "prieš {hours} val., {minutes} min."
msgstr "prieš {hours} val., {minutes} min"
#: searx/answerers/random.py:68
msgid "Generate different random values"
msgstr "Generuoja įvairias atsitiktines reikšmes"
msgstr "Generuoja įvairias atsitiktinius skaičius"
#: searx/answerers/statistics.py:37
#, python-brace-format
msgid "Compute {func} of the arguments"
msgstr "Apskaičiuoti argumentų {func}"
msgstr "Apskaičiuoti {func} iš argumentų"
#: searx/engines/boardreader.py:108
#, python-brace-format
msgid "Posted by {author}"
msgstr "Paskelbė {author}"
msgstr ""
#: searx/engines/openstreetmap.py:155
msgid "Show route in map .."
@@ -677,7 +675,7 @@ msgstr "balsai"
#: searx/engines/radio_browser.py:164
msgid "clicks"
msgstr "spustelėjimai"
msgstr "paspaudimai"
#: searx/engines/semantic_scholar.py:141
#, python-brace-format
@@ -685,8 +683,8 @@ msgid ""
"{numCitations} citations from the year {firstCitationVelocityYear} to "
"{lastCitationVelocityYear}"
msgstr ""
"{numCitations} citatos nuo {firstCitationVelocityYear} metų iki "
"{lastCitationVelocityYear} metų"
"{numCitations} citatos iš metų{firstCitationVelocityYear} to "
"{lastCitationVelocityYear}"
#: searx/engines/tineye.py:42
msgid ""
@@ -694,22 +692,21 @@ msgid ""
"format. TinEye only supports images that are JPEG, PNG, GIF, BMP, TIFF or"
" WebP."
msgstr ""
"Nepavyko perskaityti tos nuotraukos URL. Taip gali būti dėl nepalaikomo "
"failo formato. TinEye palaiko tik JPEG, PNG, GIF, BMP, TIFF arba WebP "
"nuotraukas."
"Nepavyko perskaityti šio vaizdo URL. Taip gali būti dėl nepalaikomo failo"
" formato. TinEye palaiko tik JPEG, PNG, GIF, BMP, TIFF arba WebP vaizdus."
#: searx/engines/tineye.py:48
msgid ""
"The image is too simple to find matches. TinEye requires a basic level of"
" visual detail to successfully identify matches."
msgstr ""
"Nuotrauka pernelyg paprasta, kad galima būtų rasti atitikčių. Norint "
"sėkmingai nustatyti atitiktis, TinEye reikia bazinio vizualinių detalių "
"lygio."
"Vaizdas per paprastas, kad būtų galima rasti atitikmenų. Norint sėkmingai"
" nustatyti atitikmenis, TinEye reikalingas pagrindinis vizualinių "
"detalių lygis."
#: searx/engines/tineye.py:53
msgid "The image could not be downloaded."
msgstr "Nepavyko atsisiųsti nuotraukos."
msgstr "Nepavyko atsisiųsti vaizdo."
#: searx/engines/zlibrary.py:80
msgid "Language"
@@ -733,11 +730,11 @@ msgstr "Išfiltruoti onion rezultatus esančius Ahmia juodajame sąraše."
#: searx/plugins/calculator.py:25
msgid "Calculator"
msgstr "Skaičiuotuvas"
msgstr ""
#: searx/plugins/calculator.py:26
msgid "Parses and solves mathematical expressions."
msgstr "Nagrinėja ir sprendžia matematinius reiškinius."
msgstr ""
#: searx/plugins/hash_plugin.py:33
msgid "Hash plugin"
@@ -748,12 +745,10 @@ msgid ""
"Converts strings to different hash digests. Available functions: md5, "
"sha1, sha224, sha256, sha384, sha512."
msgstr ""
"Konvertuoja eilutes į skirtingas maišos reikšmes. Prieinamos funkcijos: md5, "
"sha1, sha224, sha256, sha384, sha512."
#: searx/plugins/hash_plugin.py:63
msgid "hash digest"
msgstr "maišos reikšmė"
msgstr "maišos santrauka"
#: searx/plugins/hostnames.py:119
msgid "Hostnames plugin"
@@ -771,7 +766,7 @@ msgstr "Begalinis slinkimas"
msgid ""
"Automatically loads the next page when scrolling to bottom of the current"
" page"
msgstr "Automatiškai įkelia kitą puslapį, kai slenkama į esamo puslapio apačią"
msgstr "Automatiškai įkelti kitą puslapį, kai nuslenkama į esamo puslapio apačią"
#: searx/plugins/oa_doi_rewrite.py:54
msgid "Open Access DOI rewrite"
@@ -807,7 +802,7 @@ msgstr "Tavo naudotojo agentas (user-agent) yra: "
#: searx/plugins/time_zone.py:33
msgid "Timezones plugin"
msgstr "Laiko juostų įskiepis"
msgstr ""
#: searx/plugins/time_zone.py:34
msgid "Display the current time on different time zones."
@@ -1549,7 +1544,7 @@ msgstr "Tema"
#: searx/templates/simple/preferences/theme.html:14
msgid "Change the layout of SearXNG"
msgstr "Keisti SearXNG išdėstymą"
msgstr ""
#: searx/templates/simple/preferences/theme.html:19
msgid "Theme style"
@@ -1577,15 +1572,15 @@ msgstr "Keisti išdėstymo kalbą"
#: searx/templates/simple/preferences/urlformatting.html:2
msgid "URL formatting"
msgstr "URL formatavimas"
msgstr ""
#: searx/templates/simple/preferences/urlformatting.html:8
msgid "Pretty"
msgstr "Gražus"
msgstr ""
#: searx/templates/simple/preferences/urlformatting.html:13
msgid "Full"
msgstr "Visas"
msgstr ""
#: searx/templates/simple/preferences/urlformatting.html:18
msgid "Host"
@@ -1632,7 +1627,7 @@ msgstr "Tipas"
#: searx/templates/simple/result_templates/file.html:67
msgid "Download"
msgstr "Atsisiųsti"
msgstr ""
#: searx/templates/simple/result_templates/images.html:53
msgid "Resolution"
@@ -1644,7 +1639,7 @@ msgstr ""
#: searx/templates/simple/result_templates/images.html:56
msgid "original format"
msgstr "pradinis formatas"
msgstr ""
#: searx/templates/simple/result_templates/images.html:64
msgid "View source"
@@ -1668,7 +1663,7 @@ msgstr "Versija"
#: searx/templates/simple/result_templates/packages.html:18
msgid "Maintainer"
msgstr "Prižiūrėtojas"
msgstr ""
#: searx/templates/simple/result_templates/packages.html:24
msgid "Updated at"
@@ -1693,7 +1688,7 @@ msgstr "Projektas"
#: searx/templates/simple/result_templates/packages.html:55
msgid "Project homepage"
msgstr "Projekto internetinė svetainė"
msgstr ""
#: searx/templates/simple/result_templates/paper.html:8
msgid "Published date"
@@ -2377,3 +2372,4 @@ msgstr "slėpti vaizdo įrašą"
#~ msgid "Engine"
#~ msgstr "Sistema"

View File

@@ -24,17 +24,16 @@ msgstr ""
"Project-Id-Version: PROJECT VERSION\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2026-07-15 15:45+0000\n"
"PO-Revision-Date: 2026-08-21 20:54+0000\n"
"PO-Revision-Date: 2026-05-19 12:08+0000\n"
"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
"Language: nb_NO\n"
"Language-Team: Norwegian Bokmål <https://translate.codeberg.org/projects/"
"searxng/searxng/nb_NO/>\n"
"Language-Team: Norwegian Bokmål "
"<https://translate.codeberg.org/projects/searxng/searxng/nb_NO/>\n"
"Plural-Forms: nplurals=2; plural=n != 1;\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.18.0\n"
"X-Generator: Weblate 2026.8.1\n"
#. CONSTANT_NAMES['NO_SUBGROUPING']
#: searx/searxng.msg
@@ -1646,7 +1645,7 @@ msgstr "Oppløsning"
#: searx/templates/simple/result_templates/images.html:55
msgid "Image formats"
msgstr "Bildeformater"
msgstr ""
#: searx/templates/simple/result_templates/images.html:56
msgid "original format"
@@ -2319,3 +2318,4 @@ msgstr "skjul video"
#~ msgid "Engine"
#~ msgstr "Søkemotor"

View File

@@ -19,21 +19,20 @@
# lukisko <lukisko@noreply.codeberg.org>, 2026.
msgid ""
msgstr ""
"Project-Id-Version: searx\n"
"Project-Id-Version: searx\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2026-07-15 15:45+0000\n"
"PO-Revision-Date: 2026-08-21 20:55+0000\n"
"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
"PO-Revision-Date: 2026-07-08 00:07+0000\n"
"Last-Translator: lukisko <lukisko@noreply.codeberg.org>\n"
"Language: sk\n"
"Language-Team: Slovak <https://translate.codeberg.org/projects/searxng/"
"searxng/sk/>\n"
"Plural-Forms: nplurals=4; plural=(n % 1 == 0 && n == 1 ? 0 : n % 1 == 0 && "
"n >= 2 && n <= 4 ? 1 : n % 1 != 0 ? 2: 3);\n"
"Language-Team: Slovak "
"<https://translate.codeberg.org/projects/searxng/searxng/sk/>\n"
"Plural-Forms: nplurals=4; plural=(n % 1 == 0 && n == 1 ? 0 : n % 1 == 0 "
"&& n >= 2 && n <= 4 ? 1 : n % 1 != 0 ? 2: 3);\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.18.0\n"
"X-Generator: Weblate 2026.8.1\n"
#. CONSTANT_NAMES['NO_SUBGROUPING']
#: searx/searxng.msg
@@ -408,7 +407,7 @@ msgstr "Dážď so snehom"
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy sleet and thunder"
msgstr "Silný dážď so snehom a hromy"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
@@ -423,7 +422,7 @@ msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy sleet"
msgstr "Silný dážď so snehom"
msgstr ""
#. WEATHER_CONDITIONS
#: searx/searxng.msg
@@ -2402,3 +2401,4 @@ msgstr "skryť video"
#~ msgid "Engine"
#~ msgstr "Vyhľadávač"

View File

@@ -39,7 +39,7 @@ msgstr ""
"Project-Id-Version: searx\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2026-07-15 15:45+0000\n"
"PO-Revision-Date: 2026-08-21 20:55+0000\n"
"PO-Revision-Date: 2026-07-17 12:20+0000\n"
"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
"Language-Team: Turkish <https://translate.codeberg.org/projects/searxng/"
"searxng/tr/>\n"
@@ -48,7 +48,7 @@ msgstr ""
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"
"X-Generator: Weblate 2026.8.1\n"
"X-Generator: Weblate 2026.6.1\n"
"Generated-By: Babel 2.18.0\n"
#. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -89,7 +89,7 @@ msgstr "görseller"
#. CATEGORY_NAMES['VIDEOS']
#: searx/searxng.msg
msgid "videos"
msgstr "videolar"
msgstr "görüntüler"
#. CATEGORY_NAMES['RADIO']
#: searx/engines/radio_browser.py:160 searx/searxng.msg
@@ -119,7 +119,7 @@ msgstr "harita"
#. CATEGORY_NAMES['ONIONS']
#: searx/searxng.msg
msgid "onions"
msgstr "onionlar (tor)"
msgstr "soğanlar"
#. CATEGORY_NAMES['SCIENCE']
#: searx/searxng.msg
@@ -239,12 +239,12 @@ msgstr "Rutubet"
#. WEATHER_TERMS['MAX TEMP.']
#: searx/searxng.msg
msgid "Max temp."
msgstr "Maks. Sıcaklık"
msgstr "Maks Sıcaklık.."
#. WEATHER_TERMS['MIN TEMP.']
#: searx/searxng.msg
msgid "Min temp."
msgstr "Min. Sıcaklık"
msgstr "Min Sıcaklık"
#. WEATHER_TERMS['MORNING']
#: searx/searxng.msg
@@ -269,7 +269,7 @@ msgstr "Basınç"
#. WEATHER_TERMS['SUNRISE']
#: searx/searxng.msg
msgid "Sunrise"
msgstr "Gün doğumu"
msgstr "gündoğumu"
#. WEATHER_TERMS['SUNSET']
#: searx/searxng.msg
@@ -364,7 +364,7 @@ msgstr "Yağmur"
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy rain and thunder"
msgstr "Şiddetli yağmur ve gök gürültüsü"
msgstr "Sağanak yağmur ve fırtına"
#. WEATHER_CONDITIONS
#: searx/searxng.msg
@@ -379,7 +379,7 @@ msgstr "Şiddetli sağanak yağış"
#. WEATHER_CONDITIONS
#: searx/searxng.msg
msgid "Heavy rain"
msgstr "Şiddetli Yağmur"
msgstr "Sağanak Yağmur"
#. WEATHER_CONDITIONS
#: searx/searxng.msg
@@ -1778,11 +1778,11 @@ msgstr "Dosya Sayısı"
#: searx/templates/simple/result_templates/videos.html:6
msgid "show video"
msgstr "videoyu göster"
msgstr "görüntüyü göster"
#: searx/templates/simple/result_templates/videos.html:6
msgid "hide video"
msgstr "videoyu gizle"
msgstr "görüntüyü gizle"
#~ msgid "Engine time (sec)"
#~ msgstr "Motor süresi (san)"

View File

@@ -484,14 +484,14 @@ def pre_request():
if k not in sxng_request.form:
sxng_request.form[k] = v
try:
if sxng_request.form.get('preferences'):
preferences.parse_encoded_data(sxng_request.form['preferences'])
else:
if sxng_request.form.get('preferences'):
preferences.parse_encoded_data(sxng_request.form['preferences'])
else:
try:
preferences.parse_dict(sxng_request.form)
except Exception as e: # pylint: disable=broad-except
logger.exception(e, exc_info=True)
sxng_request.errors.append(gettext('Invalid settings'))
except Exception as e: # pylint: disable=broad-except
logger.exception(e, exc_info=True)
sxng_request.errors.append(gettext('Invalid settings'))
# language is defined neither in settings nor in preferences
# use browser headers

View File

@@ -1,34 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Shared methods for accessing Wikidata."""
import typing as t
from urllib.parse import urlencode
from searx.network import get, post
from searx.utils import gen_useragent
# SPARQL
SPARQL_ENDPOINT_URL = "https://query.wikidata.org/sparql"
SPARQL_EXPLAIN_URL = "https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain"
def send_wikidata_query(query: str, method: str = "GET", **kwargs: dict[str, t.Any]) -> dict[str, t.Any]:
if method == "GET":
# query will be cached by wikidata
http_response = get(
SPARQL_ENDPOINT_URL + "?" + urlencode({"query": query}), headers=get_wikidata_headers(), **kwargs
)
else:
# query won't be cached by wikidata
http_response = post(SPARQL_ENDPOINT_URL, data={"query": query}, headers=get_wikidata_headers(), **kwargs)
http_response.raise_for_status()
return http_response.json()
def get_wikidata_headers() -> dict[str, str]:
# user agent: https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits
return {
"Accept": "application/sparql-results+json",
"User-Agent": gen_useragent(),
}

View File

@@ -1,572 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-class-docstring
"""Fetch property names from :origin:`searx/engines/wikidata.py` engine."""
import typing as t
from dateutil.parser import isoparse
from babel.dates import format_datetime, format_date, format_time, get_datetime_format
from searx.external_urls import get_earth_coordinates_url, get_external_url
from searx.data import WikiDataPropertiesType, WikiDataUnitType
from searx.wikidata import send_wikidata_query
# SERVICE wikibase:mwapi : https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual/MWAPI
# SERVICE wikibase:label: https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_Label#Manual_Label_SERVICE
# https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates
# https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format#Data_model
# optimization:
# * https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/query_optimization
# * https://github.com/blazegraph/database/wiki/QueryHints
QUERY_TEMPLATE = """
SELECT ?item ?itemLabel ?itemDescription ?lat ?long %SELECT%
WHERE
{
SERVICE wikibase:mwapi {
bd:serviceParam wikibase:endpoint "www.wikidata.org";
wikibase:api "EntitySearch";
wikibase:limit 1;
mwapi:search "%QUERY%";
mwapi:language "%LANGUAGE%".
?item wikibase:apiOutputItem mwapi:item.
}
hint:Prior hint:runFirst "true".
%WHERE%
SERVICE wikibase:label {
bd:serviceParam wikibase:language "%LANGUAGE%,en".
?item rdfs:label ?itemLabel .
?item schema:description ?itemDescription .
%WIKIBASE_LABELS%
}
}
GROUP BY ?item ?itemLabel ?itemDescription ?lat ?long %GROUP_BY%
"""
# Get the calendar names and the property names
QUERY_PROPERTY_NAMES = """
SELECT ?item ?name
WHERE {
{
SELECT ?item
WHERE { ?item wdt:P279* wd:Q12132 }
} UNION {
VALUES ?item { %ATTRIBUTES% }
}
OPTIONAL { ?item rdfs:label ?name. }
}
"""
class WDAttribute:
def __init__(self, name: str):
self.name: str = name
def get_select(self):
return "(group_concat(distinct ?{name};separator=', ') as ?{name}s)".replace("{name}", self.name)
def get_label(self, language: str):
return get_label_for_entity(self.name, language)
def get_where(self):
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace("{name}", self.name)
def get_wikibase_label(self) -> str:
return ""
def get_group_by(self) -> str:
return ""
def get_str(self, result: dict[str, t.Any], language: str) -> str | None: # pylint: disable=unused-argument
return result.get(self.name + "s")
def __repr__(self):
return "<" + str(type(self).__name__) + ":" + self.name + ">"
class WDAmountAttribute(WDAttribute):
def get_select(self) -> str:
return "?{name} ?{name}Unit".replace("{name}", self.name)
def get_where(self):
return """ OPTIONAL { ?item p:{name} ?{name}Node .
?{name}Node rdf:type wikibase:BestRank ; ps:{name} ?{name} .
OPTIONAL { ?{name}Node psv:{name}/wikibase:quantityUnit ?{name}Unit. } }""".replace('{name}', self.name)
def get_group_by(self) -> str:
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name)
unit: str | None = result.get(self.name + "Unit")
if unit is not None:
unit = unit.replace("http://www.wikidata.org/entity/", "")
return str(value) + " " + get_label_for_entity(unit, language)
return value
class WDArticle(WDAttribute):
def __init__(self, language: str, kwargs: dict[str, t.Any] | None = None):
super().__init__("wikipedia")
self.language: str = language
self.kwargs: dict[str, t.Any] = kwargs or {}
def get_label(self, language: str):
# language parameter is ignored
return "Wikipedia ({language})".replace("{language}", self.language)
def get_select(self):
return "?article{language} ?articleName{language}".replace("{language}", self.language)
def get_where(self):
return """OPTIONAL { ?article{language} schema:about ?item ;
schema:inLanguage "{language}" ;
schema:isPartOf <https://{language}.wikipedia.org/> ;
schema:name ?articleName{language} . }""".replace('{language}', self.language)
def get_group_by(self):
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
key = "article{language}".replace("{language}", self.language)
return result.get(key)
class WDLabelAttribute(WDAttribute):
def get_select(self):
return "(group_concat(distinct ?{name}Label;separator=', ') as ?{name}Labels)".replace("{name}", self.name)
def get_where(self):
return "OPTIONAL { ?item wdt:{name} ?{name} . }".replace("{name}", self.name)
def get_wikibase_label(self) -> str:
return "?{name} rdfs:label ?{name}Label .".replace("{name}", self.name)
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
return result.get(self.name + "Labels")
class WDURLAttribute(WDAttribute):
HTTP_WIKIMEDIA_IMAGE: str = "http://commons.wikimedia.org/wiki/Special:FilePath/"
def __init__(
self,
name: str,
url_id: str | None = None,
url_path_prefix: str | None = None,
kwargs: dict[str, t.Any] | None = None,
):
"""
:param url_id: ID matching one key in ``external_urls.json`` for
converting IDs to full URLs.
:param url_path_prefix: Path prefix if the values are of format
``account@domain``. If provided, value are rewritten to
``https://<domain><url_path_prefix><account>``. For example::
WDURLAttribute('P4033', url_path_prefix='/@')
Adds Property `P4033 <https://www.wikidata.org/wiki/Property:P4033>`_
to the wikidata query. This field might return for example
``libreoffice@fosstodon.org`` and the URL built from this is then:
- account: ``libreoffice``
- domain: ``fosstodon.org``
- result url: https://fosstodon.org/@libreoffice
"""
super().__init__(name)
self.url_id: str | None = url_id
self.url_path_prefix: str | None = url_path_prefix
self.kwargs: dict[str, t.Any] = kwargs or {}
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name + "s")
if not value:
return None
value = value.split(",")[0]
if self.url_id:
url_id = self.url_id
if value.startswith(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE):
value = value[len(WDURLAttribute.HTTP_WIKIMEDIA_IMAGE) :]
url_id = "wikimedia_image"
return get_external_url(url_id, value)
if self.url_path_prefix:
[account, domain] = [x.strip("@ ") for x in value.rsplit("@", 1)]
return f"https://{domain}{self.url_path_prefix}{account}"
return value
class WDGeoAttribute(WDAttribute):
def get_label(self, language: str):
return "OpenStreetMap"
def get_select(self):
return "?{name}Lat ?{name}Long".replace("{name}", self.name)
def get_where(self):
return """OPTIONAL { ?item p:{name}/psv:{name} [
wikibase:geoLatitude ?{name}Lat ;
wikibase:geoLongitude ?{name}Long ] }""".replace('{name}', self.name)
def get_group_by(self):
return self.get_select()
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
latitude: str | None = result.get(self.name + "Lat")
longitude: str | None = result.get(self.name + "Long")
if latitude and longitude:
return latitude + " " + longitude
return None
def get_geo_url(self, result: dict[str, t.Any], osm_zoom: int = 19) -> str | None:
latitude: str | None = result.get(self.name + "Lat")
longitude: str | None = result.get(self.name + "Long")
if latitude and longitude:
return get_earth_coordinates_url(latitude, longitude, osm_zoom)
return None
class WDImageAttribute(WDURLAttribute):
def __init__(self, name: str, url_id: str | None = None, priority: int = 100):
super().__init__(name, url_id)
self.priority: int = priority
class WDDateAttribute(WDAttribute):
def get_select(self):
return "?{name} ?{name}timePrecision ?{name}timeZone ?{name}timeCalendar".replace("{name}", self.name)
def get_where(self):
# To remove duplicate, add
# FILTER NOT EXISTS { ?item p:{name}/psv:{name}/wikibase:timeValue ?{name}bis FILTER (?{name}bis < ?{name}) }
# this filter is too slow, so the response function ignore duplicate results
# (see the seen_entities variable)
return """OPTIONAL { ?item p:{name}/psv:{name} [
wikibase:timeValue ?{name} ;
wikibase:timePrecision ?{name}timePrecision ;
wikibase:timeTimezone ?{name}timeZone ;
wikibase:timeCalendarModel ?{name}timeCalendar ] . }
hint:Prior hint:rangeSafe true;""".replace('{name}', self.name)
def get_group_by(self):
return self.get_select()
def format_8(self, value: str, locale: str) -> str: # pylint: disable=unused-argument
# precision: less than a year
return value
def format_9(self, value: str, locale: str) -> str:
year = int(value)
# precision: year
if year < 1584:
if year < 0:
return str(year - 1)
return str(year)
timestamp = isoparse(value)
return format_date(timestamp, format="yyyy", locale=locale)
def format_10(self, value: str, locale: str) -> str:
# precision: month
timestamp = isoparse(value)
return format_date(timestamp, format="MMMM y", locale=locale)
def format_11(self, value: str, locale: str) -> str:
# precision: day
timestamp = isoparse(value)
return format_date(timestamp, format="full", locale=locale)
def format_13(self, value: str, locale: str) -> str:
timestamp = isoparse(value)
# precision: minute
return (
get_datetime_format("medium", locale=locale)
.replace("'", "")
.replace("{0}", format_time(timestamp, "full", tzinfo=None, locale=locale))
.replace("{1}", format_date(timestamp, "short", locale=locale))
)
def format_14(self, value: str, locale: str) -> str:
# precision: second.
return format_datetime(isoparse(value), format="full", locale=locale)
DATE_FORMAT: dict[str, tuple[str, int]] = {
"0": ("format_8", 1000000000),
"1": ("format_8", 100000000),
"2": ("format_8", 10000000),
"3": ("format_8", 1000000),
"4": ("format_8", 100000),
"5": ("format_8", 10000),
"6": ("format_8", 1000),
"7": ("format_8", 100),
"8": ("format_8", 10),
"9": ("format_9", 1), # year
"10": ("format_10", 1), # month
"11": ("format_11", 0), # day
"12": ("format_13", 0), # hour (not supported by babel, display minute)
"13": ("format_13", 0), # minute
"14": ("format_14", 0), # second
}
def get_str(self, result: dict[str, t.Any], language: str) -> str | None:
value: str | None = result.get(self.name)
if value == "" or value is None:
return None
_p: str = result.get(self.name + "timePrecision") or "1"
date_format = WDDateAttribute.DATE_FORMAT.get(_p)
if date_format is not None:
format_method = getattr(self, date_format[0])
precision: int = date_format[1]
try:
if precision >= 1:
_t = value.split("-")
if value.startswith("-"):
value = "-" + _t[1]
else:
value = _t[0]
return format_method(value, language)
except Exception: # pylint: disable=broad-except
return value
return value
WDAttrType = (
WDAttribute
| WDAmountAttribute
| WDArticle
| WDLabelAttribute
| WDURLAttribute
| WDGeoAttribute
| WDImageAttribute
| WDDateAttribute
)
WDAttrList = list[WDAttrType]
_WIKIDATA_PROPERTIES_OVERRIDE: WikiDataPropertiesType = {
"P434": "MusicBrainz",
"P435": "MusicBrainz",
"P436": "MusicBrainz",
"P966": "MusicBrainz",
"P345": "IMDb",
"P2397": "YouTube",
"P1651": "YouTube",
"P2002": "Twitter",
"P2013": "Facebook",
"P2003": "Instagram",
"P4033": "Mastodon",
"P11947": "Lemmy",
"P12622": "PeerTube",
}
"""Custom hardcoded property names for some Wikidata IDs. Only used if the
name Wikidata assigned isn't user-friendly."""
def fetch_properties(units: dict[str, WikiDataUnitType]) -> WikiDataPropertiesType:
properties = _WIKIDATA_PROPERTIES_OVERRIDE
# WIKIDATA_PROPERTIES : add unit symbols
for k, v in units.items():
properties[k] = v["symbol"]
# WIKIDATA_PROPERTIES : add property labels
wikidata_property_names: list[str] = []
for attribute in get_attributes("en"):
if type(attribute) in (WDAttribute, WDAmountAttribute, WDURLAttribute, WDDateAttribute, WDLabelAttribute):
if attribute.name not in properties:
wikidata_property_names.append("wd:" + attribute.name)
query = QUERY_PROPERTY_NAMES.replace("%ATTRIBUTES%", " ".join(wikidata_property_names))
kwargs: dict[str, t.Any] = {"timeout": 60}
json_response = send_wikidata_query(query, **kwargs)
for result in json_response.get("results", {}).get("bindings", {}):
name_field = result.get("name")
if not name_field:
continue
name = name_field["value"]
lang = name_field["xml:lang"]
entity_id = result["item"]["value"].replace("http://www.wikidata.org/entity/", "")
if name:
prop = properties.get(entity_id) or {}
prop[lang] = name.capitalize() # pyright: ignore[reportIndexIssue]
properties[entity_id] = prop
else:
properties[entity_id] = name.capitalize()
return properties
def get_attributes(language: str):
# pylint: disable=too-many-statements
attributes: WDAttrList = []
def add_value(name: str):
attributes.append(WDAttribute(name))
def add_amount(name: str):
attributes.append(WDAmountAttribute(name))
def add_label(name: str):
attributes.append(WDLabelAttribute(name))
def add_url(name: str, url_id: str | None = None, url_path_prefix: str | None = None, **kwargs: dict[str, t.Any]):
attributes.append(WDURLAttribute(name, url_id, url_path_prefix, kwargs))
def add_image(name: str, url_id: str | None = None, priority: int = 1):
attributes.append(WDImageAttribute(name, url_id, priority))
def add_date(name: str):
attributes.append(WDDateAttribute(name))
# Dates
for p in [
"P571", # inception date
"P576", # dissolution date
"P580", # start date
"P582", # end date
"P569", # date of birth
"P570", # date of death
"P619", # date of spacecraft launch
"P620",
]: # date of spacecraft landing
add_date(p)
for p in [
"P27", # country of citizenship
"P495", # country of origin
"P17", # country
"P159",
]: # headquarters location
add_label(p)
# Places
for p in [
"P36", # capital
"P35", # head of state
"P6", # head of government
"P122", # basic form of government
"P37",
]: # official language
add_label(p)
add_value("P1082") # population
add_amount("P2046") # area
add_amount("P281") # postal code
add_label("P38") # currency
add_amount("P2048") # height (building)
# Media
for p in [
"P400", # platform (videogames, computing)
"P50", # author
"P170", # creator
"P57", # director
"P175", # performer
"P178", # developer
"P162", # producer
"P176", # manufacturer
"P58", # screenwriter
"P272", # production company
"P264", # record label
"P123", # publisher
"P449", # original network
"P750", # distributed by
"P86",
]: # composer
add_label(p)
add_date("P577") # publication date
add_label("P136") # genre (music, film, artistic...)
add_label("P364") # original language
add_value("P212") # ISBN-13
add_value("P957") # ISBN-10
add_label("P275") # copyright license
add_label("P277") # programming language
add_value("P348") # version
add_label("P840") # narrative location
# Languages
add_value("P1098") # number of speakers
add_label("P282") # writing system
add_label("P1018") # language regulatory body
add_value("P218") # language code (ISO 639-1)
# Other
add_label("P169") # ceo
add_label("P112") # founded by
add_label("P1454") # legal form (company, organization)
add_label("P137") # operator (service, facility, ...)
add_label("P1029") # crew members (tripulation)
add_label("P225") # taxon name
add_value("P274") # chemical formula
add_label("P1346") # winner (sports, contests, ...)
add_value("P1120") # number of deaths
add_value("P498") # currency code (ISO 4217)
# URL
kwargs: dict[str, t.Any] = {"official": True}
add_url("P856", **kwargs) # official website
attributes.append(WDArticle(language)) # wikipedia (user language)
if not language.startswith("en"):
attributes.append(WDArticle("en")) # wikipedia (english)
add_url("P1324") # source code repository
add_url("P1581") # blog
add_url("P434", url_id="musicbrainz_artist")
add_url("P435", url_id="musicbrainz_work")
add_url("P436", url_id="musicbrainz_release_group")
add_url("P966", url_id="musicbrainz_label")
add_url("P345", url_id="imdb_id")
add_url("P2397", url_id="youtube_channel")
add_url("P1651", url_id="youtube_video")
add_url("P2002", url_id="twitter_profile")
add_url("P2013", url_id="facebook_profile")
add_url("P2003", url_id="instagram_profile")
# Fediverse
add_url("P4033", url_path_prefix="/@") # Mastodon user
add_url("P11947", url_path_prefix="/c/") # Lemmy community
add_url("P12622", url_path_prefix="/c/") # PeerTube channel
# Map
attributes.append(WDGeoAttribute("P625"))
# Image
add_image("P15", priority=1, url_id="wikimedia_image") # route map
add_image("P242", priority=2, url_id="wikimedia_image") # locator map
add_image("P154", priority=3, url_id="wikimedia_image") # logo
add_image("P18", priority=4, url_id="wikimedia_image") # image
add_image("P41", priority=5, url_id="wikimedia_image") # flag
add_image("P2716", priority=6, url_id="wikimedia_image") # collage
add_image("P2910", priority=7, url_id="wikimedia_image") # icon
return attributes
def get_label_for_entity(entity_id: str, language: str) -> str:
# only import properties locally to prevent cyclic import when initializing WIKIDATA_PROPERTIES
from searx.data import WIKIDATA_PROPERTIES # pylint: disable=import-outside-toplevel
property_name = WIKIDATA_PROPERTIES.get(entity_id)
if property_name is None:
return entity_id
if isinstance(property_name, str):
return property_name
if name := property_name.get(language):
return name
if name := property_name.get(language.split("-")[0]):
return name
if name := property_name.get("en"):
return name
return entity_id

View File

@@ -11,7 +11,7 @@ __all__ = ["convert_from_si", "convert_to_si", "symbol_to_si"]
import collections
from searx import data
from searx.wikidata import send_wikidata_query
from searx.engines import wikidata
class Beaufort:
@@ -142,6 +142,7 @@ def units_by_si_name(si_name):
# build the catalog ..
for item in symbol_to_si():
item_si_name = item[pos_si_name]
item_symbol = item[pos_symbol]
@@ -265,13 +266,14 @@ ORDER BY ?item DESC(?rank) ?symbol
"""
def fetch_units() -> dict[str, data.WikiDataUnitType]:
def fetch_units():
"""Fetch units from Wikidata. Function is used to update persistence of
:py:obj:`searx.data.WIKIDATA_UNITS`."""
results = collections.OrderedDict()
response = send_wikidata_query(SARQL_REQUEST)
response = wikidata.send_wikidata_query(SARQL_REQUEST)
for unit in response['results']['bindings']:
symbol = unit['symbol']['value']
name = unit['item']['value'].rsplit('/', 1)[1]
si_name = unit.get('tosiUnit', {}).get('value', '')

View File

@@ -16,7 +16,6 @@ import json
from searx.locales import LOCALE_NAMES, locales_initialize
from searx.engines import wikidata, set_loggers
from searx.data.currencies import CurrenciesDB
from searx.wikidata import send_wikidata_query
set_loggers(wikidata, 'wikidata')
locales_initialize()
@@ -91,7 +90,7 @@ def add_currency_label(db, label, iso4217, language):
def wikidata_request_result_iterator(request):
result = send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL), timeout=20)
result = wikidata.send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL), timeout=20)
if result is not None:
yield from result['results']['bindings']

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Fetch website description from websites and from
:origin:`searx/wikidata.py`.
:origin:`searx/engines/wikidata.py` engine.
Output file: :origin:`searx/data/engine_descriptions.json`.
@@ -17,7 +17,6 @@ from lxml.html import fromstring
import searx.engines
from searx.engines import wikidata, set_loggers
from searx.wikidata import send_wikidata_query
from searx.utils import extract_text
from searx.locales import LOCALE_NAMES, locales_initialize, match_locale
from searx import searx_dir
@@ -201,6 +200,7 @@ def initialize():
locale2lang = {"nl-BE": "nl"}
for sxng_ui_lang in LOCALE_NAMES:
sxng_ui_alias = locale2lang.get(sxng_ui_lang, sxng_ui_lang)
wiki_lang = None
@@ -225,7 +225,7 @@ def initialize():
def fetch_wikidata_descriptions():
print("Fetching wikidata descriptions")
searx.network.set_timeout_for_thread(60)
result = send_wikidata_query(
result = wikidata.send_wikidata_query(
SPARQL_DESCRIPTION.replace("%IDS%", IDS).replace("%LANGUAGES_SPARQL%", LANGUAGES_SPARQL)
)
if not result:
@@ -249,7 +249,7 @@ def fetch_wikidata_descriptions():
def fetch_wikipedia_descriptions():
print("Fetching wikipedia descriptions")
result = send_wikidata_query(
result = wikidata.send_wikidata_query(
SPARQL_WIKIPEDIA_ARTICLE.replace("%IDS%", IDS).replace("%LANGUAGES_SPARQL%", LANGUAGES_SPARQL)
)
if not result:
@@ -307,6 +307,7 @@ def fetch_website_description(engine_name: str, website: str):
previous_count: int = 0
for lang in languages:
if lang in descriptions[engine_name]:
continue

View File

@@ -50,7 +50,6 @@ from searx.engines import wikidata, set_loggers
from searx.sxng_locales import sxng_locales
from searx.engines.openstreetmap import get_key_rank, VALUE_TO_LINK
from searx.data import data_dir
from searx.wikidata import send_wikidata_query
DATA_FILE = data_dir / 'osm_keys_tags.json'
@@ -103,7 +102,7 @@ def get_preset_keys():
def get_keys():
results = get_preset_keys()
response = send_wikidata_query(SPARQL_KEYS_REQUEST)
response = wikidata.send_wikidata_query(SPARQL_KEYS_REQUEST)
for key in response['results']['bindings']:
keys = key['key']['value'].split(':')[1:]
@@ -149,7 +148,7 @@ def get_keys():
def get_tags():
results = collections.OrderedDict()
response = send_wikidata_query(SPARQL_TAGS_REQUEST)
response = wikidata.send_wikidata_query(SPARQL_TAGS_REQUEST)
for tag in response['results']['bindings']:
tag_names = tag['tag']['value'].split(':')[1].split('=')
if len(tag_names) == 2:
@@ -205,6 +204,7 @@ def optimize_keys(data):
if __name__ == '__main__':
set_timeout_for_thread(60)
result = {
'keys': optimize_keys(get_keys()),

View File

@@ -1,29 +0,0 @@
#!/usr/bin/env python
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Fetch units and property names from :origin:`searx/engines/wikidata.py` engine.
Output files: (:origin:`CI Update data <.github/workflows/data-update.yml>`).
- :origin:`searx/data/wikidata_units.json`
- :origin:`searx/data/wikidata_properties.json`
"""
import json
from searx.engines import wikidata, set_loggers
from searx.data import data_dir
from searx.wikidata_properties import fetch_properties
from searx.wikidata_units import fetch_units
UNITS_DATA_FILE = data_dir / 'wikidata_units.json'
PROPERTIES_DATA_FILE = data_dir / 'wikidata_properties.json'
set_loggers(wikidata, 'wikidata')
if __name__ == '__main__':
units = fetch_units()
with UNITS_DATA_FILE.open('w', encoding="utf8") as f:
json.dump(units, f, indent=4, sort_keys=True, ensure_ascii=False)
properties = fetch_properties(units)
with PROPERTIES_DATA_FILE.open('w', encoding="utf8") as f:
json.dump(properties, f, indent=4, sort_keys=True, ensure_ascii=False)

View File

@@ -0,0 +1,22 @@
#!/usr/bin/env python
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Fetch units from :origin:`searx/engines/wikidata.py` engine.
Output file: :origin:`searx/data/wikidata_units.json` (:origin:`CI Update data
... <.github/workflows/data-update.yml>`).
"""
import json
from searx.engines import wikidata, set_loggers
from searx.data import data_dir
from searx.wikidata_units import fetch_units
DATA_FILE = data_dir / 'wikidata_units.json'
set_loggers(wikidata, 'wikidata')
if __name__ == '__main__':
with DATA_FILE.open('w', encoding="utf8") as f:
json.dump(fetch_units(), f, indent=4, sort_keys=True, ensure_ascii=False)

View File

@@ -26,8 +26,7 @@ data.all() {
build_msg DATA "update searx/data/ahmia_blacklist.txt"
python searxng_extra/update/update_ahmia_blacklist.py
build_msg DATA "update searx/data/wikidata_units.json"
build_msg DATA "update searx/data/wikidata_properties.json"
python searxng_extra/update/update_wikidata.py
python searxng_extra/update/update_wikidata_units.py
build_msg DATA "update searx/data/currencies.json"
python searxng_extra/update/update_currencies.py
build_msg DATA "update searx/data/external_bangs.json"