[fix] wikidata: increase wikidata queries timeout

The big queries for initializing and updating the currencies take longer than
the default of the wikidata engine, which is only 3sec.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

Makefile

@@ -75,7 +75,7 @@ test.shell:
 # wrap ./manage script
 
 MANAGE += weblate.translations.commit weblate.push.translations
-MANAGE += data.all data.traits data.useragents data.locales
+MANAGE += data.all data.traits data.useragents data.locales data.currencies
 MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
 MANAGE += docker.build docker.push docker.buildx
 MANAGE += gecko.driver

searx/data/currencies.json

@@ -1557,7 +1557,7 @@
             "ar": "بيزو دومنيكاني",
             "bg": "Доминиканско песо",
             "ca": "peso dominicà",
-            "cs": "Dominikánské peso",
+            "cs": "dominikánské peso",
             "cy": "Peso Dominica",
             "de": "dominikanischer Peso",
             "en": "Dominican peso",
@@ -1582,7 +1582,7 @@
             "pt": "peso dominicano",
             "ro": "peso dominican",
             "ru": "доминиканское песо",
-            "sk": "Dominikánske peso",
+            "sk": "dominikánske peso",
             "sl": "dominikanski peso",
             "sr": "доминикански пезос",
             "sv": "Dominikansk peso",
@@ -1809,7 +1809,7 @@
             "fi": "Fidžin dollari",
             "fr": "dollar des Fidji",
             "ga": "dollar Fhidsí",
-            "gl": "Dólar fidxiano",
+            "gl": "Dólar fijiano",
             "he": "דולר פיג'י",
             "hr": "Fidžijski dolar",
             "hu": "fidzsi dollár",
@@ -1921,7 +1921,7 @@
             "ar": "لاري جورجي",
             "bg": "Грузинско лари",
             "ca": "lari",
-            "cs": "gruzínské lari",
+            "cs": "gruzínský lari",
             "da": "Lari",
             "de": "georgischer Lari",
             "en": "Georgian lari",
@@ -3308,7 +3308,7 @@
             "ar": "درهم مغربي",
             "bg": "Марокански дирхам",
             "ca": "dírham marroquí",
-            "cs": "Marocký dirham",
+            "cs": "marocký dirham",
             "cy": "Dirham Moroco",
             "de": "Marokkanischer Dirham",
             "en": "Moroccan dirham",
@@ -3776,6 +3776,7 @@
             "en": "Malaysian ringgit",
             "eo": "malajzia ringito",
             "es": "ringgit",
+            "et": "Ringgit",
             "eu": "Ringgit",
             "fi": "Malesian ringgit",
             "fr": "ringgit",
@@ -3885,12 +3886,13 @@
             "bg": "Нигерийска найра",
             "ca": "naira",
             "cs": "nigerijská naira",
+            "cy": "naira",
             "de": "Naira",
             "en": "naira",
             "eo": "niĝeria najro",
             "es": "naira",
             "fi": "Nigerian naira",
-            "fr": "Naira",
+            "fr": "naira",
             "ga": "naira",
             "gl": "Naira",
             "he": "נאירה",
@@ -4412,7 +4414,7 @@
             "ar": "غواراني باراغواي",
             "ca": "guaraní",
             "cs": "paraguayský guaraní",
-            "cy": "Gwarani Paragwâi",
+            "cy": "Gwarani Paragwái",
             "de": "Paraguayischer Guaraní",
             "en": "Paraguayan guaraní",
             "eo": "paragvaja gvaranio",
@@ -6284,6 +6286,7 @@
             "es": "Oro de Zimbabue",
             "fi": "Zimbabwen kulta",
             "fr": "or du Zimbabwe",
+            "hr": "zimbabveanski gold",
             "id": "Zimbabwe Gold",
             "it": "Zimbabwe Gold",
             "ja": "ジンバブエ・ゴールド",
@@ -6459,7 +6462,6 @@
         "NT$": "TWD",
         "NZ$": "NZD",
         "Nfk": "ERN",
-        "Noha heinen krasss": "EUR",
         "Nu": "BTN",
         "N₨": "NPR",
         "P": "BWP",
@@ -8146,7 +8148,6 @@
         "dólar dos barbados": "BBD",
         "dólar dos estados unidos": "USD",
         "dólar estadounidense": "USD",
-        "dólar fidxiano": "FJD",
         "dólar fijiano": "FJD",
         "dólar fixiano": "FJD",
         "dólar fiyiano": "FJD",
@@ -8832,7 +8833,7 @@
         "gvinejski franak": "GNF",
         "gvinejski frank": "GNF",
         "gvinėjos frankas": "GNF",
-        "gwarani paragwâi": "PYG",
+        "gwarani paragwái": "PYG",
         "gy$": "GYD",
         "gyd": "GYD",
         "ĝibraltara pundo": "GIP",
@@ -10186,6 +10187,8 @@
         "naira nigeriana": "NGN",
         "naira nigeriano": "NGN",
         "naira nigérian": "NGN",
+        "naïra": "NGN",
+        "naïra nigérian": "NGN",
         "najro": "NGN",
         "nakfa": "ERN",
         "nakfa eritrea": "ERN",
@@ -12442,6 +12445,7 @@
         "zimbabva dolaro": "ZWL",
         "zimbabva oro": "ZWG",
         "zimbabve altını": "ZWG",
+        "zimbabveanski gold": "ZWG",
         "zimbabvejski dolar": "ZWL",
         "zimbabvejski gold": "ZWG",
         "zimbabwaanse dollar": "ZWL",
@@ -13060,6 +13064,7 @@
         "джибутийский франк": "DJF",
         "джибутски франк": "DJF",
         "дин": "RSD",
+        "дин.": "RSD",
         "динар": [
             "TND",
             "RSD",
@@ -14304,7 +14309,9 @@
         "الروبل": "RUB",
         "الروبل الروسي": "RUB",
         "الروبية الهندية": "INR",
+        "الريال السعودي": "SAR",
         "الشيكل الإسرائيلي الجديد": "ILS",
+        "العملة السعودية": "SAR",
         "العملة الموريتانية": "MRU",
         "الفرنك الجيبوتي": "DJF",
         "الفرنك السويسري": "CHF",
@@ -14465,6 +14472,8 @@
         "روفيه": "MVR",
         "روفيه مالديفية": "MVR",
         "ريال": "YER",
+        "ريال السعودية": "SAR",
+        "ريال المملكة العربية السعودية": "SAR",
         "ريال إيراني": "IRR",
         "ريال برازيلي": "BRL",
         "ريال سعودي": "SAR",
@@ -14473,6 +14482,7 @@
         "ريال قطري": "QAR",
         "ريال كمبودي": "KHR",
         "ريال يمني": "YER",
+        "ريالات": "SAR",
         "رينغيت": "MYR",
         "رينغيت ماليزي": "MYR",
         "ریال": [
@@ -14525,6 +14535,8 @@
         "شيلينغ تنزاني": "TZS",
         "شيلينغ كيني": "KES",
         "شيلينغ كينيي": "KES",
+        "عملة السعودية": "SAR",
+        "عملة المملكة العربية السعودية": "SAR",
         "غواراني": "PYG",
         "غواراني باراغواي": "PYG",
         "غيلدر الأنتيل الهولندية": "ANG",
@@ -15778,7 +15790,6 @@
         "香港元": "HKD",
         "﷼": [
             "YER",
-            "SAR",
             "IRR"
         ],
         "﷼'": "YER"

searx/engines/bing.py

@@ -223,7 +223,7 @@ def fetch_traits(engine_traits: EngineTraits):
         'da': 'dk',  # da --> da-dk
     }
 
-    for href in eval_xpath(dom, '//div[@id="language-section"]//li/a/@href'):
+    for href in eval_xpath(dom, '//div[@id="language-section-content"]//div[@class="languageItem"]/a/@href'):
         eng_lang = parse_qs(urlparse(href).query)['setlang'][0]
         babel_lang = map_lang.get(eng_lang, eng_lang)
         try:
@@ -253,7 +253,7 @@ def fetch_traits(engine_traits: EngineTraits):
     map_market_codes = {
         'zh-hk': 'en-hk',  # not sure why, but at M$ this is the market code for Hongkong
     }
-    for href in eval_xpath(dom, '//div[@id="region-section"]//li/a/@href'):
+    for href in eval_xpath(dom, '//div[@id="region-section-content"]//div[@class="regionItem"]/a/@href'):
         cc_tag = parse_qs(urlparse(href).query)['cc'][0]
         if cc_tag == 'clear':
             engine_traits.all_locale = cc_tag

searx/engines/wikidata.py

@@ -158,13 +158,13 @@ def get_label_for_entity(entity_id, language):
     return name
 
 
-def send_wikidata_query(query, method='GET'):
+def send_wikidata_query(query, method='GET', **kwargs):
     if method == 'GET':
         # query will be cached by wikidata
-        http_response = get(SPARQL_ENDPOINT_URL + '?' + urlencode({'query': query}), headers=get_headers())
+        http_response = get(SPARQL_ENDPOINT_URL + '?' + urlencode({'query': query}), headers=get_headers(), **kwargs)
     else:
         # query won't be cached by wikidata
-        http_response = post(SPARQL_ENDPOINT_URL, data={'query': query}, headers=get_headers())
+        http_response = post(SPARQL_ENDPOINT_URL, data={'query': query}, headers=get_headers(), **kwargs)
     if http_response.status_code != 200:
         logger.debug('SPARQL endpoint error %s', http_response.content.decode())
     logger.debug('request time %s', str(http_response.elapsed))
@@ -808,7 +808,7 @@ def init(engine_settings=None):  # pylint: disable=unused-argument
             if attribute.name not in WIKIDATA_PROPERTIES:
                 wikidata_property_names.append("wd:" + attribute.name)
     query = QUERY_PROPERTY_NAMES.replace('%ATTRIBUTES%', " ".join(wikidata_property_names))
-    jsonresponse = send_wikidata_query(query)
+    jsonresponse = send_wikidata_query(query, timeout=20)
     for result in jsonresponse.get('results', {}).get('bindings', {}):
         name = result['name']['value']
         lang = result['name']['xml:lang']

searx/sxng_locales.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-'''List of SearXNG's locale codes.
+'''List of SearXNG's locale codes used for the search language/region.
 
 .. hint::
 
@@ -11,14 +11,9 @@
 sxng_locales = (
     ('af', 'Afrikaans', '', 'Afrikaans', '\U0001f310'),
     ('ar', 'العربية', '', 'Arabic', '\U0001f310'),
-    ('ar-SA', 'العربية', 'المملكة العربية السعودية', 'Arabic', '\U0001f1f8\U0001f1e6'),
-    ('be', 'Беларуская', '', 'Belarusian', '\U0001f310'),
     ('bg', 'Български', '', 'Bulgarian', '\U0001f310'),
-    ('bg-BG', 'Български', 'България', 'Bulgarian', '\U0001f1e7\U0001f1ec'),
     ('ca', 'Català', '', 'Catalan', '\U0001f310'),
     ('cs', 'Čeština', '', 'Czech', '\U0001f310'),
-    ('cs-CZ', 'Čeština', 'Česko', 'Czech', '\U0001f1e8\U0001f1ff'),
-    ('cy', 'Cymraeg', '', 'Welsh', '\U0001f310'),
     ('da', 'Dansk', '', 'Danish', '\U0001f310'),
     ('da-DK', 'Dansk', 'Danmark', 'Danish', '\U0001f1e9\U0001f1f0'),
     ('de', 'Deutsch', '', 'German', '\U0001f310'),
@@ -26,7 +21,6 @@ sxng_locales = (
     ('de-CH', 'Deutsch', 'Schweiz', 'German', '\U0001f1e8\U0001f1ed'),
     ('de-DE', 'Deutsch', 'Deutschland', 'German', '\U0001f1e9\U0001f1ea'),
     ('el', 'Ελληνικά', '', 'Greek', '\U0001f310'),
-    ('el-GR', 'Ελληνικά', 'Ελλάδα', 'Greek', '\U0001f1ec\U0001f1f7'),
     ('en', 'English', '', 'English', '\U0001f310'),
     ('en-AU', 'English', 'Australia', 'English', '\U0001f1e6\U0001f1fa'),
     ('en-CA', 'English', 'Canada', 'English', '\U0001f1e8\U0001f1e6'),
@@ -35,20 +29,13 @@ sxng_locales = (
     ('en-IN', 'English', 'India', 'English', '\U0001f1ee\U0001f1f3'),
     ('en-NZ', 'English', 'New Zealand', 'English', '\U0001f1f3\U0001f1ff'),
     ('en-PH', 'English', 'Philippines', 'English', '\U0001f1f5\U0001f1ed'),
-    ('en-PK', 'English', 'Pakistan', 'English', '\U0001f1f5\U0001f1f0'),
-    ('en-SG', 'English', 'Singapore', 'English', '\U0001f1f8\U0001f1ec'),
     ('en-US', 'English', 'United States', 'English', '\U0001f1fa\U0001f1f8'),
     ('en-ZA', 'English', 'South Africa', 'English', '\U0001f1ff\U0001f1e6'),
     ('es', 'Español', '', 'Spanish', '\U0001f310'),
     ('es-AR', 'Español', 'Argentina', 'Spanish', '\U0001f1e6\U0001f1f7'),
     ('es-CL', 'Español', 'Chile', 'Spanish', '\U0001f1e8\U0001f1f1'),
-    ('es-CO', 'Español', 'Colombia', 'Spanish', '\U0001f1e8\U0001f1f4'),
     ('es-ES', 'Español', 'España', 'Spanish', '\U0001f1ea\U0001f1f8'),
     ('es-MX', 'Español', 'México', 'Spanish', '\U0001f1f2\U0001f1fd'),
-    ('es-PE', 'Español', 'Perú', 'Spanish', '\U0001f1f5\U0001f1ea'),
-    ('et', 'Eesti', '', 'Estonian', '\U0001f310'),
-    ('et-EE', 'Eesti', 'Eesti', 'Estonian', '\U0001f1ea\U0001f1ea'),
-    ('fa', 'فارسی', '', 'Persian', '\U0001f310'),
     ('fi', 'Suomi', '', 'Finnish', '\U0001f310'),
     ('fi-FI', 'Suomi', 'Suomi', 'Finnish', '\U0001f1eb\U0001f1ee'),
     ('fr', 'Français', '', 'French', '\U0001f310'),
@@ -56,14 +43,10 @@ sxng_locales = (
     ('fr-CA', 'Français', 'Canada', 'French', '\U0001f1e8\U0001f1e6'),
     ('fr-CH', 'Français', 'Suisse', 'French', '\U0001f1e8\U0001f1ed'),
     ('fr-FR', 'Français', 'France', 'French', '\U0001f1eb\U0001f1f7'),
-    ('ga', 'Gaeilge', '', 'Irish', '\U0001f310'),
-    ('gd', 'Gàidhlig', '', 'Scottish Gaelic', '\U0001f310'),
     ('gl', 'Galego', '', 'Galician', '\U0001f310'),
     ('he', 'עברית', '', 'Hebrew', '\U0001f1ee\U0001f1f1'),
-    ('hi', 'हिन्दी', '', 'Hindi', '\U0001f310'),
     ('hr', 'Hrvatski', '', 'Croatian', '\U0001f310'),
     ('hu', 'Magyar', '', 'Hungarian', '\U0001f310'),
-    ('hu-HU', 'Magyar', 'Magyarország', 'Hungarian', '\U0001f1ed\U0001f1fa'),
     ('id', 'Indonesia', '', 'Indonesian', '\U0001f310'),
     ('id-ID', 'Indonesia', 'Indonesia', 'Indonesian', '\U0001f1ee\U0001f1e9'),
     ('it', 'Italiano', '', 'Italian', '\U0001f310'),
@@ -71,13 +54,9 @@ sxng_locales = (
     ('it-IT', 'Italiano', 'Italia', 'Italian', '\U0001f1ee\U0001f1f9'),
     ('ja', '日本語', '', 'Japanese', '\U0001f310'),
     ('ja-JP', '日本語', '日本', 'Japanese', '\U0001f1ef\U0001f1f5'),
-    ('kn', 'ಕನ್ನಡ', '', 'Kannada', '\U0001f310'),
     ('ko', '한국어', '', 'Korean', '\U0001f310'),
     ('ko-KR', '한국어', '대한민국', 'Korean', '\U0001f1f0\U0001f1f7'),
     ('lt', 'Lietuvių', '', 'Lithuanian', '\U0001f310'),
-    ('lv', 'Latviešu', '', 'Latvian', '\U0001f310'),
-    ('ml', 'മലയാളം', '', 'Malayalam', '\U0001f310'),
-    ('mr', 'मराठी', '', 'Marathi', '\U0001f310'),
     ('nb', 'Norsk Bokmål', '', 'Norwegian Bokmål', '\U0001f310'),
     ('nb-NO', 'Norsk Bokmål', 'Norge', 'Norwegian Bokmål', '\U0001f1f3\U0001f1f4'),
     ('nl', 'Nederlands', '', 'Dutch', '\U0001f310'),
@@ -89,22 +68,16 @@ sxng_locales = (
     ('pt-BR', 'Português', 'Brasil', 'Portuguese', '\U0001f1e7\U0001f1f7'),
     ('pt-PT', 'Português', 'Portugal', 'Portuguese', '\U0001f1f5\U0001f1f9'),
     ('ro', 'Română', '', 'Romanian', '\U0001f310'),
-    ('ro-RO', 'Română', 'România', 'Romanian', '\U0001f1f7\U0001f1f4'),
     ('ru', 'Русский', '', 'Russian', '\U0001f310'),
     ('ru-RU', 'Русский', 'Россия', 'Russian', '\U0001f1f7\U0001f1fa'),
     ('sk', 'Slovenčina', '', 'Slovak', '\U0001f310'),
-    ('sl', 'Slovenščina', '', 'Slovenian', '\U0001f310'),
     ('sq', 'Shqip', '', 'Albanian', '\U0001f310'),
     ('sv', 'Svenska', '', 'Swedish', '\U0001f310'),
     ('sv-SE', 'Svenska', 'Sverige', 'Swedish', '\U0001f1f8\U0001f1ea'),
-    ('ta', 'தமிழ்', '', 'Tamil', '\U0001f310'),
-    ('te', 'తెలుగు', '', 'Telugu', '\U0001f310'),
     ('th', 'ไทย', '', 'Thai', '\U0001f310'),
-    ('th-TH', 'ไทย', 'ไทย', 'Thai', '\U0001f1f9\U0001f1ed'),
     ('tr', 'Türkçe', '', 'Turkish', '\U0001f310'),
     ('tr-TR', 'Türkçe', 'Türkiye', 'Turkish', '\U0001f1f9\U0001f1f7'),
     ('uk', 'Українська', '', 'Ukrainian', '\U0001f310'),
-    ('ur', 'اردو', '', 'Urdu', '\U0001f310'),
     ('vi', 'Tiếng Việt', '', 'Vietnamese', '\U0001f310'),
     ('zh', '中文', '', 'Chinese', '\U0001f310'),
     ('zh-CN', '中文', '中国', 'Chinese', '\U0001f1e8\U0001f1f3'),

searxng_extra/update/update_currencies.py

@@ -92,8 +92,7 @@ def add_currency_label(db, label, iso4217, language):
 
 
 def wikidata_request_result_iterator(request):
-    wikidata.timeout = 30  # github CI has longer timeouts
-    result = wikidata.send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
+    result = wikidata.send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL), timeout=20)
     if result is not None:
         yield from result['results']['bindings']
 

searxng_extra/update/update_engine_traits.py

@@ -28,7 +28,7 @@ from searx.enginelib.traits import EngineTraitsMap
 languages_file = Path(searx_dir) / 'sxng_locales.py'
 languages_file_header = """\
 # SPDX-License-Identifier: AGPL-3.0-or-later
-'''List of SearXNG's locale codes.
+'''List of SearXNG's locale codes used for the search language/region.
 
 .. hint::
 
@@ -101,8 +101,8 @@ def fetch_traits_map():
 def filter_locales(traits_map: EngineTraitsMap):
     """Filter language & region tags by a threshold."""
 
-    min_eng_per_region = 18
-    min_eng_per_lang = 20
+    min_eng_per_region = 22
+    min_eng_per_lang = 24
 
     _ = {}
     for eng in traits_map.values():

utils/lib_sxng_data.sh

@@ -8,6 +8,7 @@ data.:
   traits    : update searx/data/engine_traits.json & searx/sxng_locales.py
   useragents: update searx/data/useragents.json with the most recent versions of Firefox
   locales   : update searx/data/locales.json from babel
+  currencies: update searx/data/currencies.json from wikidata
 EOF
 }
 
@@ -59,3 +60,12 @@ data.locales() {
     )
     dump_return $?
 }
+
+data.currencies(){
+    (   set -e
+        pyenv.activate
+        build_msg DATA "update searx/data/currencies.json"
+        python searxng_extra/update/update_currencies.py
+    )
+    dump_return $?
+}