jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2026-08-01 21:01:24 +00:00
Code Issues Projects Releases Wiki Activity

[mod] update_engine_descriptions.py - revision of the script (#6309)

Browse Source 
* [mod] update_engine_descriptions.py - revision of the script (#6309)

The script was a bit outdated, type hints were missing, and the strings were
converted to double quotation marks

The `descriptions.items()` are sorted to avoid assigning the references to
seen_descriptions randomly (should reduce unnecessary diffs in the future).

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

* [data] update searx.data - update_engine_descriptions.py (#6309)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

---------

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
2026-06-24 12:10:28 +02:00
committed by GitHub
parent f9f3d089ec
commit e3126b89e6
2 changed files with 2899 additions and 2938 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5549
searx/data/engine_descriptions.json
View File

File diff suppressed because it is too large Load Diff

288
searxng_extra/update/update_engine_descriptions.py
View File

@@ -15,25 +15,26 @@ from os.path import join
from lxml.html import fromstring from lxml.html import fromstring
import searx.engines
from searx.engines import wikidata, set_loggers from searx.engines import wikidata, set_loggers
from searx.utils import extract_text, searxng_useragent from searx.utils import extract_text
from searx.locales import LOCALE_NAMES, locales_initialize, match_locale from searx.locales import LOCALE_NAMES, locales_initialize, match_locale
from searx import searx_dir from searx import searx_dir
from searx.utils import gen_useragent from searx.utils import gen_useragent
import searx.search import searx.search
import searx.network import searx.network
from searx.data import data_dir from searx.data import data_dir, ENGINE_DESCRIPTIONS
DATA_FILE = data_dir / 'engine_descriptions.json' DATA_FILE = data_dir / "engine_descriptions.json"
set_loggers(wikidata, 'wikidata') set_loggers(wikidata, "wikidata")
locales_initialize() locales_initialize()
# you can run the query in https://query.wikidata.org # you can run the query in https://query.wikidata.org
# replace %IDS% by Wikidata entities separated by spaces with the prefix wd: # replace %IDS% by Wikidata entities separated by spaces with the prefix wd:
# for example wd:Q182496 wd:Q1540899 # for example wd:Q182496 wd:Q1540899
# replace %LANGUAGES_SPARQL% by languages # replace %LANGUAGES_SPARQL% by languages
SPARQL_WIKIPEDIA_ARTICLE = """ SPARQL_WIKIPEDIA_ARTICLE: str = """
SELECT DISTINCT ?item ?name ?article ?lang SELECT DISTINCT ?item ?name ?article ?lang
WHERE { WHERE {
hint:Query hint:optimizer "None". hint:Query hint:optimizer "None".
@@ -48,7 +49,7 @@ WHERE {
ORDER BY ?item ?lang ORDER BY ?item ?lang
""" """
SPARQL_DESCRIPTION = """ SPARQL_DESCRIPTION: str = """
SELECT DISTINCT ?item ?itemDescription SELECT DISTINCT ?item ?itemDescription
WHERE { WHERE {
VALUES ?item { %IDS% } VALUES ?item { %IDS% }
@@ -58,43 +59,43 @@ WHERE {
ORDER BY ?itemLang ORDER BY ?itemLang
""" """
NOT_A_DESCRIPTION = [ NOT_A_DESCRIPTION: list[str] = [
'web site', "web site",
'site web', "site web",
'komputa serĉilo', "komputa serĉilo",
'interreta serĉilo', "interreta serĉilo",
'bilaketa motor', "bilaketa motor",
'web search engine', "web search engine",
'wikimedia täpsustuslehekülg', "wikimedia täpsustuslehekülg",
] ]
SKIP_ENGINE_SOURCE = [ SKIP_ENGINE_SOURCE: list[tuple[str, str]] = [
# fmt: off # fmt: off
('gitlab', 'wikidata') ("gitlab", "wikidata")
# descriptions are about wikipedia disambiguation pages # descriptions are about wikipedia disambiguation pages
# fmt: on # fmt: on
] ]
WIKIPEDIA_LANGUAGES = {} WIKIPEDIA_LANGUAGES: dict[str, str] = {}
LANGUAGES_SPARQL = '' LANGUAGES_SPARQL: str = ""
IDS = None IDS: str = ""
WIKIPEDIA_LANGUAGE_VARIANTS = {'zh_Hant': 'zh-tw'} WIKIPEDIA_LANGUAGE_VARIANTS: dict[str, str] = {"zh_Hant": "zh-tw"}
# descriptions[engine][language] = [description, source]
descriptions: dict[str, dict[str, list[str]]] = {}
wd_to_engine_name: dict[str, set[str]] = {}
descriptions = {} def normalize_description(description: str):
wd_to_engine_name = {}
def normalize_description(description):
for c in [chr(c) for c in range(0, 31)]: for c in [chr(c) for c in range(0, 31)]:
description = description.replace(c, ' ') description = description.replace(c, " ")
description = ' '.join(description.strip().split()) description = " ".join(description.strip().split())
return description return description
def update_description(engine_name, lang, description, source, replace=True): def update_description(engine_name: str, lang: str, description: str, source: str, replace: bool = True) -> None:
if not isinstance(description, str): if not isinstance(description, str):
return return # pyright: ignore[reportUnreachable]
description = normalize_description(description) description = normalize_description(description)
if description.lower() == engine_name.lower(): if description.lower() == engine_name.lower():
return return
@@ -102,54 +103,71 @@ def update_description(engine_name, lang, description, source, replace=True):
return return
if (engine_name, source) in SKIP_ENGINE_SOURCE: if (engine_name, source) in SKIP_ENGINE_SOURCE:
return return
if ' ' not in description: if " " not in description:
# skip unique word description (like "website") # skip unique word description (like "website")
return return
if replace or lang not in descriptions[engine_name]: if replace or lang not in descriptions[engine_name]:
descriptions[engine_name][lang] = [description, source] descriptions[engine_name][lang] = [description, source]
def get_wikipedia_summary(wikipedia_url, searxng_locale): def get_wikipedia_summary(wikipedia_url: str, searxng_locale: str):
# get the REST API URL from the HTML URL # get the REST API URL from the HTML URL
# Headers # Headers
headers = {'User-Agent': searxng_useragent()} headers = {
"User-Agent": gen_useragent(),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate",
"DNT": "1",
"Upgrade-Insecure-Requests": "1",
"Sec-GPC": "1",
"Cache-Control": "max-age=0",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "same-origin",
"Sec-Fetch-User": "?1",
}
if searxng_locale in WIKIPEDIA_LANGUAGE_VARIANTS: if searxng_locale in WIKIPEDIA_LANGUAGE_VARIANTS:
headers['Accept-Language'] = WIKIPEDIA_LANGUAGE_VARIANTS.get(searxng_locale) headers["Accept-Language"] = WIKIPEDIA_LANGUAGE_VARIANTS[searxng_locale]
# URL path : from HTML URL to REST API URL # URL path : from HTML URL to REST API URL
parsed_url = urlparse(wikipedia_url) parsed_url = urlparse(wikipedia_url)
# remove the /wiki/ prefix # remove the /wiki/ prefix
article_name = parsed_url.path.split('/wiki/')[1] article_name = parsed_url.path.split("/wiki/")[1]
# article_name is already encoded but not the / which is required for the REST API call # article_name is already encoded but not the / which is required for the REST API call
encoded_article_name = article_name.replace('/', '%2F') encoded_article_name = article_name.replace("/", "%2F")
path = '/api/rest_v1/page/summary/' + encoded_article_name path = "/api/rest_v1/page/summary/" + encoded_article_name
wikipedia_rest_url = parsed_url._replace(path=path).geturl() wikipedia_rest_url = parsed_url._replace(path=path).geturl()
try: try:
response = searx.network.get(wikipedia_rest_url, headers=headers, timeout=10) response = searx.network.get(wikipedia_rest_url, headers=headers, timeout=10)
response.raise_for_status() response.raise_for_status()
except Exception as e: # pylint: disable=broad-except except Exception as e: # pylint: disable=broad-except
print(" ", wikipedia_url, e) print(" get_wikipedia_summary: ", wikipedia_rest_url, e)
return None return None
api_result = json.loads(response.text) api_result = json.loads(response.text)
return api_result.get('extract') return api_result.get("extract")
def get_website_description(url, lang1, lang2=None): def get_website_description(url: str, lang1: str | None, lang2: str | None = None):
headers = { headers = {
'User-Agent': gen_useragent(), "User-Agent": gen_useragent(),
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
'DNT': '1', "Accept-Encoding": "gzip, deflate",
'Upgrade-Insecure-Requests': '1', "DNT": "1",
'Sec-GPC': '1', "Upgrade-Insecure-Requests": "1",
'Cache-Control': 'max-age=0', "Sec-GPC": "1",
"Cache-Control": "max-age=0",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "same-origin",
"Sec-Fetch-User": "?1",
} }
if lang1 is not None: if lang1 is not None:
lang_list = [lang1] lang_list = [lang1]
if lang2 is not None: if lang2 is not None:
lang_list.append(lang2) lang_list.append(lang2)
headers['Accept-Language'] = f'{",".join(lang_list)};q=0.8' headers["Accept-Language"] = f'{",".join(lang_list)};q=0.8'
try: try:
response = searx.network.get(url, headers=headers, timeout=10) response = searx.network.get(url, headers=headers, timeout=10)
response.raise_for_status() response.raise_for_status()
@@ -167,7 +185,7 @@ def get_website_description(url, lang1, lang2=None):
if not description: if not description:
description = extract_text(html.xpath('/html/head/title')) description = extract_text(html.xpath('/html/head/title'))
lang = extract_text(html.xpath('/html/@lang')) lang = extract_text(html.xpath('/html/@lang'))
if lang is None and len(lang1) > 0: if lang is None and lang1 and len(lang1) > 0:
lang = lang1 lang = lang1
lang = lang or 'en' lang = lang or 'en'
lang = lang.split('_')[0] lang = lang.split('_')[0]
@@ -178,15 +196,15 @@ def get_website_description(url, lang1, lang2=None):
def initialize(): def initialize():
global IDS, LANGUAGES_SPARQL global IDS, LANGUAGES_SPARQL
searx.search.initialize() searx.search.initialize()
wikipedia_engine = searx.engines.engines['wikipedia'] wikipedia_engine = searx.engines.engines["wikipedia"]
locale2lang = {'nl-BE': 'nl'} locale2lang = {"nl-BE": "nl"}
for sxng_ui_lang in LOCALE_NAMES: for sxng_ui_lang in LOCALE_NAMES:
sxng_ui_alias = locale2lang.get(sxng_ui_lang, sxng_ui_lang) sxng_ui_alias = locale2lang.get(sxng_ui_lang, sxng_ui_lang)
wiki_lang = None wiki_lang = None
if sxng_ui_alias in wikipedia_engine.traits.custom['WIKIPEDIA_LANGUAGES']: if sxng_ui_alias in wikipedia_engine.traits.custom["WIKIPEDIA_LANGUAGES"]:
wiki_lang = sxng_ui_alias wiki_lang = sxng_ui_alias
if not wiki_lang: if not wiki_lang:
wiki_lang = wikipedia_engine.traits.get_language(sxng_ui_alias) wiki_lang = wikipedia_engine.traits.get_language(sxng_ui_alias)
@@ -195,70 +213,84 @@ def initialize():
continue continue
WIKIPEDIA_LANGUAGES[sxng_ui_lang] = wiki_lang WIKIPEDIA_LANGUAGES[sxng_ui_lang] = wiki_lang
LANGUAGES_SPARQL = ', '.join(f"'{l}'" for l in set(WIKIPEDIA_LANGUAGES.values())) LANGUAGES_SPARQL = ", ".join(f"'{l}'" for l in set(WIKIPEDIA_LANGUAGES.values()))
for engine_name, engine in searx.engines.engines.items(): for engine_name, engine in searx.engines.engines.items():
descriptions[engine_name] = {} descriptions[engine_name] = {}
wikidata_id = getattr(engine, "about", {}).get('wikidata_id') if engine.about.wikidata_id:
if wikidata_id is not None: wd_to_engine_name.setdefault(engine.about.wikidata_id, set()).add(engine_name)
wd_to_engine_name.setdefault(wikidata_id, set()).add(engine_name)
IDS = ' '.join(list(map(lambda wd_id: 'wd:' + wd_id, wd_to_engine_name.keys()))) IDS = " ".join(list(map(lambda wd_id: "wd:" + wd_id, wd_to_engine_name.keys())))
def fetch_wikidata_descriptions(): def fetch_wikidata_descriptions():
print('Fetching wikidata descriptions') print("Fetching wikidata descriptions")
searx.network.set_timeout_for_thread(60) searx.network.set_timeout_for_thread(60)
result = wikidata.send_wikidata_query( result = wikidata.send_wikidata_query(
SPARQL_DESCRIPTION.replace('%IDS%', IDS).replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL) SPARQL_DESCRIPTION.replace("%IDS%", IDS).replace("%LANGUAGES_SPARQL%", LANGUAGES_SPARQL)
) )
if result is not None: if not result:
for binding in result['results']['bindings']: print("ERROR: fetching wikiDATA descriptions - SPARQL_DESCRIPTION query without results.")
wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '') return
wikidata_lang = binding['itemDescription']['xml:lang']
desc = binding['itemDescription']['value'] for binding in result["results"]["bindings"]:
for engine_name in wd_to_engine_name[wikidata_id]: wikidata_id = binding["item"]["value"].replace("http://www.wikidata.org/entity/", "")
for searxng_locale in LOCALE_NAMES: wikidata_lang = binding["itemDescription"]["xml:lang"]
if WIKIPEDIA_LANGUAGES[searxng_locale] != wikidata_lang: desc = binding["itemDescription"]["value"]
continue for engine_name in wd_to_engine_name[wikidata_id]:
print( for searxng_locale in LOCALE_NAMES:
f" engine: {engine_name:20} / wikidata_lang: {wikidata_lang:5}", if WIKIPEDIA_LANGUAGES[searxng_locale] != wikidata_lang:
f"/ len(wikidata_desc): {len(desc)}", continue
) print(
update_description(engine_name, searxng_locale, desc, 'wikidata') f" engine: {engine_name:20} / wikidata_lang: {wikidata_lang:5}",
f"/ len(wikidata_desc): {len(desc)}",
)
update_description(engine_name, searxng_locale, desc, "wikidata")
def fetch_wikipedia_descriptions(): def fetch_wikipedia_descriptions():
print('Fetching wikipedia descriptions') print("Fetching wikipedia descriptions")
result = wikidata.send_wikidata_query( result = wikidata.send_wikidata_query(
SPARQL_WIKIPEDIA_ARTICLE.replace('%IDS%', IDS).replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL) SPARQL_WIKIPEDIA_ARTICLE.replace("%IDS%", IDS).replace("%LANGUAGES_SPARQL%", LANGUAGES_SPARQL)
) )
if result is not None: if not result:
for binding in result['results']['bindings']: print("ERROR: fetching wikiPEDIA descriptions - SPARQL_WIKIPEDIA_ARTICLE query without results.")
wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '') return
wikidata_lang = binding['name']['xml:lang']
wikipedia_url = binding['article']['value'] # for example the URL https://de.wikipedia.org/wiki/PubMed # pylint: disable=too-many-nested-blocks
for engine_name in wd_to_engine_name[wikidata_id]: for binding in result["results"]["bindings"]:
for searxng_locale in LOCALE_NAMES: wikidata_id = binding["item"]["value"].replace("http://www.wikidata.org/entity/", "")
if WIKIPEDIA_LANGUAGES[searxng_locale] != wikidata_lang: wikidata_lang = binding["name"]["xml:lang"]
continue wikipedia_url = binding["article"]["value"] # for example the URL https://de.wikipedia.org/wiki/PubMed
desc = get_wikipedia_summary(wikipedia_url, searxng_locale) for engine_name in wd_to_engine_name[wikidata_id]:
if not desc: for searxng_locale in LOCALE_NAMES:
continue if WIKIPEDIA_LANGUAGES[searxng_locale] != wikidata_lang:
print( continue
f" engine: {engine_name:20} / wikidata_lang: {wikidata_lang:5}", desc = get_wikipedia_summary(wikipedia_url, searxng_locale)
f"/ len(wikipedia_desc): {len(desc)}", if not desc:
) if descriptions.get(searxng_locale, {}).get(engine_name) is None:
update_description(engine_name, searxng_locale, desc, 'wikipedia') _descr = ENGINE_DESCRIPTIONS.get(searxng_locale, {}).get(engine_name)
if _descr is not None:
if len(_descr) == 2 and _descr[1] == 'ref':
ref_engine, ref_lang = _descr[0].split(':')
_descr = ENGINE_DESCRIPTIONS[ref_lang][ref_engine]
update_description(engine_name, searxng_locale, _descr[0], _descr[1])
continue
print(
f" engine: {engine_name:20} / wikidata_lang: {wikidata_lang:5}",
f"/ len(wikipedia_desc): {len(desc)}",
)
update_description(engine_name, searxng_locale, desc, "wikipedia")
def normalize_url(url): def normalize_url(url: str):
url = url.replace('{language}', 'en') url = url.replace("{language}", "en")
url = urlparse(url)._replace(path='/', params='', query='', fragment='').geturl() url = urlparse(url)._replace(path="/", params="", query="", fragment="").geturl()
url = url.replace('https://api.', 'https://') url = url.replace("https://api.", "https://")
return url return url
def fetch_website_description(engine_name, website): def fetch_website_description(engine_name: str, website: str):
print(f"- fetch website descr: {engine_name} / {website}") print(f"- fetch website descr: {engine_name} / {website}")
default_lang, default_description = get_website_description(website, None, None) default_lang, default_description = get_website_description(website, None, None)
@@ -268,11 +300,11 @@ def fetch_website_description(engine_name, website):
# to specify an order in where the most common languages are in front of the # to specify an order in where the most common languages are in front of the
# language list .. # language list ..
languages = ['en', 'es', 'pt', 'ru', 'tr', 'fr'] languages = ["en", "es", "pt", "ru", "tr", "fr"]
languages = languages + [l for l in LOCALE_NAMES if l not in languages] languages = languages + [l for l in LOCALE_NAMES if l not in languages]
previous_matched_lang = None previous_matched_lang: str | None = None
previous_count = 0 previous_count: int = 0
for lang in languages: for lang in languages:
@@ -307,19 +339,15 @@ def fetch_website_description(engine_name, website):
f" / fetched lang: {fetched_lang:7} / len(desc): {len(desc)}" f" / fetched lang: {fetched_lang:7} / len(desc): {len(desc)}"
) )
matched_lang = match_locale(fetched_lang, LOCALE_NAMES.keys(), fallback=lang) matched_lang = match_locale(fetched_lang, list(LOCALE_NAMES.keys())) or lang
update_description(engine_name, matched_lang, desc, website, replace=False) update_description(engine_name, matched_lang, desc, website, replace=False)
def fetch_website_descriptions(): def fetch_website_descriptions():
print('Fetching website descriptions') print("Fetching website descriptions")
for engine_name, engine in searx.engines.engines.items(): for engine_name, engine in searx.engines.engines.items():
website = getattr(engine, "about", {}).get('website') website = engine.about.website or getattr(engine, "search_url", "") or getattr(engine, "base_url", "")
if website is None and hasattr(engine, "search_url"): if website:
website = normalize_url(getattr(engine, "search_url"))
if website is None and hasattr(engine, "base_url"):
website = normalize_url(getattr(engine, "base_url"))
if website is not None:
fetch_website_description(engine_name, website) fetch_website_description(engine_name, website)
@@ -328,30 +356,32 @@ def get_engine_descriptions_filename():
def get_output(): def get_output():
"""Summary of the results, once known descriptions are not duplicated,
instead a reference is provided.
- from: ``descriptions[engine][language] = [description, source]``
- to: ``output[language][engine] = description_and_source``
``description_and_source`` can be:
- ``[description, source]``
- ``description`` (if source = "wikipedia")
- ``[f"engine:lang", "ref"]`` reference to another existing description
""" """
From descriptions[engine][language] = [description, source] output: dict[str, dict[str, list[str] | str]] = {locale: {} for locale in LOCALE_NAMES}
To seen_descriptions: dict[str, tuple[str, str]] = {}
* output[language][engine] = description_and_source for engine_name, lang_descriptions in sorted(descriptions.items()):
* description_and_source can be: for language, descr in sorted(lang_descriptions.items()):
* [description, source] if descr[0] in seen_descriptions:
* description (if source = "wikipedia") ref = seen_descriptions[descr[0]]
* [f"engine:lang", "ref"] (reference to another existing description) descr = [f"{ref[0]}:{ref[1]}", "ref"]
"""
output = {locale: {} for locale in LOCALE_NAMES}
seen_descriptions = {}
for engine_name, lang_descriptions in descriptions.items():
for language, description in lang_descriptions.items():
if description[0] in seen_descriptions:
ref = seen_descriptions[description[0]]
description = [f'{ref[0]}:{ref[1]}', 'ref']
else: else:
seen_descriptions[description[0]] = (engine_name, language) seen_descriptions[descr[0]] = (engine_name, language)
if description[1] == 'wikipedia': if descr[1] == "wikipedia":
description = description[0] descr = descr[0]
output.setdefault(language, {}).setdefault(engine_name, description) output.setdefault(language, {}).setdefault(engine_name, descr)
return output return output
@@ -363,8 +393,8 @@ def main():
fetch_website_descriptions() fetch_website_descriptions()
output = get_output() output = get_output()
with DATA_FILE.open('w', encoding='utf8') as f: with DATA_FILE.open("w", encoding="utf8") as f:
f.write(json.dumps(output, indent=1, separators=(',', ':'), sort_keys=True, ensure_ascii=False)) f.write(json.dumps(output, indent=1, separators=(",", ":"), sort_keys=True, ensure_ascii=False))
if __name__ == "__main__": if __name__ == "__main__":