From bd17544f82809ef57f09aa95dc735e7c9056717e Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 3 Aug 2021 15:11:02 +0200 Subject: [PATCH 1/4] [fix] make test.robot: firefox profile: intl.accept_languages=en make sure Firefox in test.robot asks for English pages --- searx/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searx/testing.py b/searx/testing.py index e5c1bd313..b31ba8997 100644 --- a/searx/testing.py +++ b/searx/testing.py @@ -79,7 +79,7 @@ class SearxRobotLayer(): def run_robot_tests(tests): print('Running {0} tests'.format(len(tests))) for test in tests: - with Browser('firefox', headless=True) as browser: + with Browser('firefox', headless=True, profile_preferences={'intl.accept_languages': 'en'}) as browser: test(browser) From f30d01ffabd50d7bb1a17da04e768c075bb8789d Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 3 Aug 2021 15:13:00 +0200 Subject: [PATCH 2/4] [mod] settings.yml: remove locales There are detected from the searx/translations directory --- searx/locales.py | 57 +++++++++++++++++++ searx/preferences.py | 3 +- searx/settings.yml | 47 --------------- searx/webapp.py | 17 ++---- searx_extra/update/update_currencies.py | 5 +- .../update/update_engine_descriptions.py | 3 +- tests/unit/test_settings_loader.py | 1 - 7 files changed, 70 insertions(+), 63 deletions(-) create mode 100644 searx/locales.py diff --git a/searx/locales.py b/searx/locales.py new file mode 100644 index 000000000..828f0608d --- /dev/null +++ b/searx/locales.py @@ -0,0 +1,57 @@ +from typing import List, Set +import os +import pathlib + +from babel import Locale + +LOCALE_NAMES = { + "ar": "العَرَبِيَّة (Arabic)", + "fil": "Wikang Filipino (Filipino)", + "oc": "Lenga D'òc (Occitan)", + "nl_BE": "Vlaams (Dutch, Belgium)", +} +UI_LOCALE_CODES: List[str] = [] +RTL_LOCALES: Set[str] = set() + + +def _get_name(locale, language_code): + language_name = locale.get_language_name(language_code).capitalize() + if language_name and ('a' <= language_name[0] <= 'z'): + language_name = language_name.capitalize() + terrirtory_name = locale.get_territory_name(language_code) + return language_name, terrirtory_name + + +def _get_locale_name(locale, locale_name): + native_language, native_territory = _get_name(locale, locale_name) + english_language, english_territory = _get_name(locale, 'en') + if native_territory == english_territory: + english_territory = None + if not native_territory and not english_territory: + if native_language == english_language: + return native_language + return native_language + ' (' + english_language + ')' + result = native_language + ', ' + native_territory + ' (' + english_language + if english_territory: + return result + ', ' + english_territory + ')' + return result + ')' + + +def initialize_locales(directory): + global LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES + for dirname in sorted(os.listdir(directory)): + # Based on https://flask-babel.tkte.ch/_modules/flask_babel.html#Babel.list_translations + locale_dir = os.path.join(directory, dirname, 'LC_MESSAGES') + if not os.path.isdir(locale_dir): + continue + info = LOCALE_NAMES.get(dirname) + if not info: + locale = Locale.parse(dirname) + LOCALE_NAMES[dirname] = _get_locale_name(locale, dirname) + if locale.text_direction == 'rtl': + RTL_LOCALES.add(dirname) + # + UI_LOCALE_CODES = [l.replace('_', '-') for l in LOCALE_NAMES] + + +initialize_locales(pathlib.Path(__file__).parent / 'translations') diff --git a/searx/preferences.py b/searx/preferences.py index e4a2ebecd..69832c052 100644 --- a/searx/preferences.py +++ b/searx/preferences.py @@ -11,6 +11,7 @@ from urllib.parse import parse_qs, urlencode from searx import settings, autocomplete from searx.languages import language_codes as languages +from searx.locales import LOCALE_NAMES from searx.webutils import VALID_LANGUAGE_CODE @@ -340,7 +341,7 @@ class Preferences: 'locale': EnumStringSetting( settings['ui']['default_locale'], is_locked('locale'), - choices=list(settings['locales'].keys()) + [''] + choices=list(LOCALE_NAMES.keys()) + [''] ), 'autocomplete': EnumStringSetting( settings['search']['autocomplete'], diff --git a/searx/settings.yml b/searx/settings.yml index 009ea6856..f4321b89d 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -1651,53 +1651,6 @@ engines: # chars: ' ' # keys: ['line'] -locales: - en: English - ar: العَرَبِيَّة (Arabic) - bg: Български (Bulgarian) - bo: བོད་སྐད་ (Tibetian) - ca: Català (Catalan) - cs: Čeština (Czech) - cy: Cymraeg (Welsh) - da: Dansk (Danish) - de: Deutsch (German) - el_GR: Ελληνικά (Greek_Greece) - eo: Esperanto (Esperanto) - es: Español (Spanish) - et: Eesti (Estonian) - eu: Euskara (Basque) - fa_IR: (fārsī) فارسى (Persian) - fi: Suomi (Finnish) - fil: Wikang Filipino (Filipino) - fr: Français (French) - gl: Galego (Galician) - he: עברית (Hebrew) - hr: Hrvatski (Croatian) - hu: Magyar (Hungarian) - ia: Interlingua (Interlingua) - it: Italiano (Italian) - ja: 日本語 (Japanese) - lt: Lietuvių (Lithuanian) - nl: Nederlands (Dutch) - nl_BE: Vlaams (Dutch_Belgium) - oc: Lenga D'òc (Occitan) - pl: Polski (Polish) - pt: Português (Portuguese) - pt_BR: Português (Portuguese_Brazil) - ro: Română (Romanian) - ru: Русский (Russian) - sk: Slovenčina (Slovak) - sl: Slovenski (Slovene) - sr: српски (Serbian) - sv: Svenska (Swedish) - te: తెలుగు (telugu) - ta: தமிழ் (Tamil) - tr: Türkçe (Turkish) - uk: українська мова (Ukrainian) - vi: tiếng việt (Vietnamese) - zh: 中文 (Chinese) - zh_TW: 國語 (Taiwanese Mandarin) - doi_resolvers: oadoi.org: 'https://oadoi.org/' doi.org: 'https://doi.org/' diff --git a/searx/webapp.py b/searx/webapp.py index d638dfb55..0bf1a37f5 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -106,6 +106,7 @@ from searx.flaskfix import patch_application from searx.autocomplete import search_autocomplete, backends as autocomplete_backends from searx.languages import language_codes as languages +from searx.locales import LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES from searx.search import SearchWithPlugins, initialize as search_initialize from searx.network import stream as http_stream from searx.search.checker import get_result as checker_get_result @@ -176,12 +177,6 @@ if (not werkzeug_reloader babel = Babel(app) -rtl_locales = [ - 'ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'fa_IR', 'glk', 'he', - 'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi' -] -ui_locale_codes = [l.replace('_', '-') for l in settings['locales'].keys()] - # used when translating category names _category_names = ( gettext('files'), @@ -258,7 +253,7 @@ def _get_browser_or_settings_language(req, lang_list): @babel.localeselector def get_locale(): if 'locale' in request.form\ - and request.form['locale'] in settings['locales']: + and request.form['locale'] in LOCALE_NAMES: # use locale from the form locale = request.form['locale'] locale_source = 'form' @@ -268,7 +263,7 @@ def get_locale(): locale_source = 'preferences' else: # use local from the browser - locale = _get_browser_or_settings_language(request, ui_locale_codes) + locale = _get_browser_or_settings_language(request, UI_LOCALE_CODES) locale = locale.replace('-', '_') locale_source = 'browser' @@ -463,7 +458,7 @@ def render(template_name, override_theme=None, **kwargs): kwargs['translations'] = json.dumps(get_translations(), separators=(',', ':')) locale = request.preferences.get_value('locale') - if locale in rtl_locales and 'rtl' not in kwargs: + if locale in RTL_LOCALES and 'rtl' not in kwargs: kwargs['rtl'] = True if 'current_language' not in kwargs: kwargs['current_language'] = match_language( @@ -1042,7 +1037,7 @@ def preferences(): return render( 'preferences.html', selected_categories = get_selected_categories(request.preferences, request.form), - locales = settings['locales'], + locales = LOCALE_NAMES, current_locale = request.preferences.get_value("locale"), image_proxy = image_proxy, engines_by_category = engines_by_category, @@ -1315,7 +1310,7 @@ def config(): 'engines': _engines, 'plugins': _plugins, 'instance_name': settings['general']['instance_name'], - 'locales': settings['locales'], + 'locales': LOCALE_NAMES, 'default_locale': settings['ui']['default_locale'], 'autocomplete': settings['search']['autocomplete'], 'safe_search': settings['search']['safe_search'], diff --git a/searx_extra/update/update_currencies.py b/searx_extra/update/update_currencies.py index 0cfb7a951..a572f4e9d 100755 --- a/searx_extra/update/update_currencies.py +++ b/searx_extra/update/update_currencies.py @@ -8,7 +8,8 @@ import json from sys import path from os.path import realpath, dirname, join -from searx import searx_dir, settings +from searx import searx_dir +from searx.locales import LOCALE_NAMES from searx.engines.wikidata import send_wikidata_query @@ -44,7 +45,7 @@ ORDER BY ?iso4217 ?article_name """ -LANGUAGES = settings['locales'].keys() +LANGUAGES = LOCALE_NAMES.keys() LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES))) diff --git a/searx_extra/update/update_engine_descriptions.py b/searx_extra/update/update_engine_descriptions.py index cf9007da3..37be77177 100755 --- a/searx_extra/update/update_engine_descriptions.py +++ b/searx_extra/update/update_engine_descriptions.py @@ -8,6 +8,7 @@ from lxml.html import fromstring from searx.engines.wikidata import send_wikidata_query from searx.utils import extract_text +from searx.locales import LOCALE_NAMES import searx import searx.search import searx.network @@ -35,7 +36,7 @@ WHERE { ORDER BY ?itemLang """ -LANGUAGES = searx.settings['locales'].keys() +LANGUAGES = LOCALE_NAMES.keys() LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES))) IDS = None diff --git a/tests/unit/test_settings_loader.py b/tests/unit/test_settings_loader.py index 7df64e524..a3257ee4b 100644 --- a/tests/unit/test_settings_loader.py +++ b/tests/unit/test_settings_loader.py @@ -41,7 +41,6 @@ class TestDefaultSettings(SearxTestCase): self.assertTrue(isinstance(settings['server']['port'], int)) self.assertTrue(isinstance(settings['server']['bind_address'], str)) self.assertTrue(isinstance(settings['engines'], list)) - self.assertTrue(isinstance(settings['locales'], dict)) self.assertTrue(isinstance(settings['doi_resolvers'], dict)) self.assertTrue(isinstance(settings['default_doi_resolver'], str)) From 809bf1a1056584e96b6c05bfe1aa55a9a0e11d4d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 3 Aug 2021 18:17:23 +0200 Subject: [PATCH 3/4] [mod] pylint & document searx.locales (settings.yml: remove locales) - Add ``# lint: pylint`` header to pylint this python file. - Fix issues reported by pylint. - Add source code documentation of modul searx.locales Signed-off-by: Markus Heiser --- docs/src/searx.locales.rst | 8 ++++++++ searx/locales.py | 28 ++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 docs/src/searx.locales.rst diff --git a/docs/src/searx.locales.rst b/docs/src/searx.locales.rst new file mode 100644 index 000000000..579247aff --- /dev/null +++ b/docs/src/searx.locales.rst @@ -0,0 +1,8 @@ +.. _searx.locales: + +======= +Locales +======= + +.. automodule:: searx.locales + :members: diff --git a/searx/locales.py b/searx/locales.py index 828f0608d..4dfb2b668 100644 --- a/searx/locales.py +++ b/searx/locales.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Initialize :py:obj:`LOCALE_NAMES`, :py:obj:`UI_LOCALE_CODES` and +:py:obj:`RTL_LOCALES`.""" + from typing import List, Set import os import pathlib @@ -10,8 +16,15 @@ LOCALE_NAMES = { "oc": "Lenga D'òc (Occitan)", "nl_BE": "Vlaams (Dutch, Belgium)", } +"""Mapping of locales and their description. Locales e.g. 'fr' or 'pt_BR' +(delimiter is *underline* '_')""" + UI_LOCALE_CODES: List[str] = [] +"""List of locales e.g. 'fr' or 'pt-BR' (delimiter is '-')""" + RTL_LOCALES: Set[str] = set() +"""List of *Right-To-Left* locales e.g. 'he' or 'fa_IR' (delimiter is +*underline* '_')""" def _get_name(locale, language_code): @@ -23,6 +36,11 @@ def _get_name(locale, language_code): def _get_locale_name(locale, locale_name): + """Get locale name e.g. 'Français - fr' or 'Português (Brasil) - pt-BR' + + :param locale: instance of :py:class:`Locale` + :param locale_name: name e.g. 'fr' or 'pt_BR' + """ native_language, native_territory = _get_name(locale, locale_name) english_language, english_territory = _get_name(locale, 'en') if native_territory == english_territory: @@ -38,11 +56,13 @@ def _get_locale_name(locale, locale_name): def initialize_locales(directory): - global LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES + """Initialize global names :py:obj:`LOCALE_NAMES`, :py:obj:`UI_LOCALE_CODES` and + :py:obj:`RTL_LOCALES`. + """ + global LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES # pylint: disable=global-statement for dirname in sorted(os.listdir(directory)): # Based on https://flask-babel.tkte.ch/_modules/flask_babel.html#Babel.list_translations - locale_dir = os.path.join(directory, dirname, 'LC_MESSAGES') - if not os.path.isdir(locale_dir): + if not os.path.isdir( os.path.join(directory, dirname, 'LC_MESSAGES') ): continue info = LOCALE_NAMES.get(dirname) if not info: @@ -50,7 +70,7 @@ def initialize_locales(directory): LOCALE_NAMES[dirname] = _get_locale_name(locale, dirname) if locale.text_direction == 'rtl': RTL_LOCALES.add(dirname) - # + UI_LOCALE_CODES = [l.replace('_', '-') for l in LOCALE_NAMES] From 0d20e5dfe39b371b550b888db9e6cd7a664bddbb Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Wed, 4 Aug 2021 09:50:34 +0200 Subject: [PATCH 4/4] [mod] searx/locales.py: language names based on Unicode CLDR rename "oc" to "Occitan": * https://github.com/unicode-org/cldr/blob/35.1/seed/main/oc.xml#L115 * https://oc.wikipedia.org/wiki/Occitan see https://github.com/searxng/searxng/pull/247#issuecomment-892382001 --- searx/locales.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/searx/locales.py b/searx/locales.py index 4dfb2b668..80defceb7 100644 --- a/searx/locales.py +++ b/searx/locales.py @@ -11,9 +11,7 @@ import pathlib from babel import Locale LOCALE_NAMES = { - "ar": "العَرَبِيَّة (Arabic)", - "fil": "Wikang Filipino (Filipino)", - "oc": "Lenga D'òc (Occitan)", + "oc": "Occitan", "nl_BE": "Vlaams (Dutch, Belgium)", } """Mapping of locales and their description. Locales e.g. 'fr' or 'pt_BR'