[refactor] wikidata: cache wikidata properties in searxng data

searxng_extra/update/update_currencies.py

@@ -16,6 +16,7 @@ import json
 from searx.locales import LOCALE_NAMES, locales_initialize
 from searx.engines import wikidata, set_loggers
 from searx.data.currencies import CurrenciesDB
+from searx.wikidata import send_wikidata_query
 
 set_loggers(wikidata, 'wikidata')
 locales_initialize()
@@ -90,7 +91,7 @@ def add_currency_label(db, label, iso4217, language):
 
 
 def wikidata_request_result_iterator(request):
-    result = wikidata.send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL), timeout=20)
+    result = send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL), timeout=20)
     if result is not None:
         yield from result['results']['bindings']
 

searxng_extra/update/update_engine_descriptions.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Fetch website description from websites and from
-:origin:`searx/engines/wikidata.py` engine.
+:origin:`searx/wikidata.py`.
 
 Output file: :origin:`searx/data/engine_descriptions.json`.
 
@@ -17,6 +17,7 @@ from lxml.html import fromstring
 
 import searx.engines
 from searx.engines import wikidata, set_loggers
+from searx.wikidata import send_wikidata_query
 from searx.utils import extract_text
 from searx.locales import LOCALE_NAMES, locales_initialize, match_locale
 from searx import searx_dir
@@ -200,7 +201,6 @@ def initialize():
 
     locale2lang = {"nl-BE": "nl"}
     for sxng_ui_lang in LOCALE_NAMES:
-
         sxng_ui_alias = locale2lang.get(sxng_ui_lang, sxng_ui_lang)
         wiki_lang = None
 
@@ -225,7 +225,7 @@ def initialize():
 def fetch_wikidata_descriptions():
     print("Fetching wikidata descriptions")
     searx.network.set_timeout_for_thread(60)
-    result = wikidata.send_wikidata_query(
+    result = send_wikidata_query(
         SPARQL_DESCRIPTION.replace("%IDS%", IDS).replace("%LANGUAGES_SPARQL%", LANGUAGES_SPARQL)
     )
     if not result:
@@ -249,7 +249,7 @@ def fetch_wikidata_descriptions():
 
 def fetch_wikipedia_descriptions():
     print("Fetching wikipedia descriptions")
-    result = wikidata.send_wikidata_query(
+    result = send_wikidata_query(
         SPARQL_WIKIPEDIA_ARTICLE.replace("%IDS%", IDS).replace("%LANGUAGES_SPARQL%", LANGUAGES_SPARQL)
     )
     if not result:
@@ -307,7 +307,6 @@ def fetch_website_description(engine_name: str, website: str):
     previous_count: int = 0
 
     for lang in languages:
-
         if lang in descriptions[engine_name]:
             continue
 

searxng_extra/update/update_osm_keys_tags.py

@@ -50,6 +50,7 @@ from searx.engines import wikidata, set_loggers
 from searx.sxng_locales import sxng_locales
 from searx.engines.openstreetmap import get_key_rank, VALUE_TO_LINK
 from searx.data import data_dir
+from searx.wikidata import send_wikidata_query
 
 DATA_FILE = data_dir / 'osm_keys_tags.json'
 
@@ -102,7 +103,7 @@ def get_preset_keys():
 
 def get_keys():
     results = get_preset_keys()
-    response = wikidata.send_wikidata_query(SPARQL_KEYS_REQUEST)
+    response = send_wikidata_query(SPARQL_KEYS_REQUEST)
 
     for key in response['results']['bindings']:
         keys = key['key']['value'].split(':')[1:]
@@ -148,7 +149,7 @@ def get_keys():
 
 def get_tags():
     results = collections.OrderedDict()
-    response = wikidata.send_wikidata_query(SPARQL_TAGS_REQUEST)
+    response = send_wikidata_query(SPARQL_TAGS_REQUEST)
     for tag in response['results']['bindings']:
         tag_names = tag['tag']['value'].split(':')[1].split('=')
         if len(tag_names) == 2:
@@ -204,7 +205,6 @@ def optimize_keys(data):
 
 
 if __name__ == '__main__':
-
     set_timeout_for_thread(60)
     result = {
         'keys': optimize_keys(get_keys()),

searxng_extra/update/update_wikidata.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Fetch units and property names from :origin:`searx/engines/wikidata.py` engine.
+
+Output files: (:origin:`CI Update data <.github/workflows/data-update.yml>`).
+- :origin:`searx/data/wikidata_units.json`
+- :origin:`searx/data/wikidata_properties.json`
+"""
+
+import json
+
+from searx.engines import wikidata, set_loggers
+from searx.data import data_dir
+from searx.wikidata_properties import fetch_properties
+from searx.wikidata_units import fetch_units
+
+UNITS_DATA_FILE = data_dir / 'wikidata_units.json'
+PROPERTIES_DATA_FILE = data_dir / 'wikidata_properties.json'
+set_loggers(wikidata, 'wikidata')
+
+
+if __name__ == '__main__':
+    units = fetch_units()
+    with UNITS_DATA_FILE.open('w', encoding="utf8") as f:
+        json.dump(units, f, indent=4, sort_keys=True, ensure_ascii=False)
+
+    properties = fetch_properties(units)
+    with PROPERTIES_DATA_FILE.open('w', encoding="utf8") as f:
+        json.dump(properties, f, indent=4, sort_keys=True, ensure_ascii=False)

searxng_extra/update/update_wikidata_units.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""Fetch units from :origin:`searx/engines/wikidata.py` engine.
-
-Output file: :origin:`searx/data/wikidata_units.json` (:origin:`CI Update data
-...  <.github/workflows/data-update.yml>`).
-
-"""
-
-import json
-
-from searx.engines import wikidata, set_loggers
-from searx.data import data_dir
-from searx.wikidata_units import fetch_units
-
-DATA_FILE = data_dir / 'wikidata_units.json'
-set_loggers(wikidata, 'wikidata')
-
-
-if __name__ == '__main__':
-    with DATA_FILE.open('w', encoding="utf8") as f:
-        json.dump(fetch_units(), f, indent=4, sort_keys=True, ensure_ascii=False)