[feat] engine: add neocities.org

The GSA headers that were introduced in PR #5644 unfortunately no longer
work (#6359).

The Google engines

- google.py
- google_videos.py

do not work anymore either, but we'll leave it in the code for now:

- In google.py, central functions like get_google_info(..) are provided, which
  are also used by other modules.
- We will probably need a Google HTML (and video) engine again very soon.

Related:

- https://github.com/searxng/searxng/issues/6359
- https://github.com/searxng/searxng/pull/6364

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

.github/workflows/data-update.yml

@@ -33,7 +33,6 @@ jobs:
           - update_engine_traits.py
           - update_wikidata_units.py
           - update_engine_descriptions.py
-          - update_gsa_useragents.py
 
     permissions:
       contents: write

Makefile

@@ -63,7 +63,7 @@ format: format.python format.shell
 # wrap ./manage script
 
 MANAGE += weblate.translations.commit weblate.push.translations
-MANAGE += data.all data.traits data.useragents data.gsa_useragents data.locales data.currencies
+MANAGE += data.all data.traits data.useragents data.locales data.currencies
 MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
 MANAGE += podman.build
 MANAGE += docker.build docker.buildx

searx/data/__init__.py

@@ -6,7 +6,7 @@ make data.all
 """
 # pylint: disable=invalid-name
 
-__all__ = ["ahmia_blacklist_loader", "gsa_useragents_loader", "data_dir", "get_cache"]
+__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
 
 import json
 import typing as t
@@ -63,7 +63,6 @@ lazy_globals = {
     "ENGINE_TRAITS": None,
     "LOCALES": None,
     "TRACKER_PATTERNS": TrackerPatternsDB(),
-    "GSA_USER_AGENTS": None,
 }
 
 data_json_files = {
@@ -106,24 +105,3 @@ def ahmia_blacklist_loader() -> list[str]:
     """
     with open(data_dir / 'ahmia_blacklist.txt', encoding='utf-8') as f:
         return f.read().split()
-
-
-def gsa_useragents_loader() -> list[str]:
-    """Load data from `gsa_useragents.txt` and return a list of user agents
-    suitable for Google.  The user agents are fetched by::
-
-      searxng_extra/update/update_gsa_useragents.py
-
-    This function is used by :py:mod:`searx.engines.google`.
-
-    """
-    data = lazy_globals["GSA_USER_AGENTS"]
-    if data is not None:
-        return data
-
-    log.debug("init searx.data.%s", "GSA_USER_AGENTS")
-
-    with open(data_dir / 'gsa_useragents.txt', encoding='utf-8') as f:
-        lazy_globals["GSA_USER_AGENTS"] = f.read().splitlines()
-
-    return lazy_globals["GSA_USER_AGENTS"]

searx/data/engine_traits.json

@@ -3476,6 +3476,598 @@
       "zu-ZA": "ZA"
     }
   },
+  "google cse": {
+    "all_locale": "ZZ",
+    "custom": {
+      "supported_domains": {
+        "AD": "www.google.ad",
+        "AE": "www.google.ae",
+        "AF": "www.google.com.af",
+        "AG": "www.google.com.ag",
+        "AL": "www.google.al",
+        "AM": "www.google.am",
+        "AO": "www.google.co.ao",
+        "AR": "www.google.com.ar",
+        "AS": "www.google.as",
+        "AT": "www.google.at",
+        "AU": "www.google.com.au",
+        "AZ": "www.google.az",
+        "BA": "www.google.ba",
+        "BD": "www.google.com.bd",
+        "BE": "www.google.be",
+        "BF": "www.google.bf",
+        "BG": "www.google.bg",
+        "BH": "www.google.com.bh",
+        "BI": "www.google.bi",
+        "BJ": "www.google.bj",
+        "BN": "www.google.com.bn",
+        "BO": "www.google.com.bo",
+        "BR": "www.google.com.br",
+        "BS": "www.google.bs",
+        "BT": "www.google.bt",
+        "BW": "www.google.co.bw",
+        "BY": "www.google.by",
+        "BZ": "www.google.com.bz",
+        "CA": "www.google.ca",
+        "CAT": "www.google.cat",
+        "CD": "www.google.cd",
+        "CF": "www.google.cf",
+        "CG": "www.google.cg",
+        "CH": "www.google.ch",
+        "CI": "www.google.ci",
+        "CK": "www.google.co.ck",
+        "CL": "www.google.cl",
+        "CM": "www.google.cm",
+        "CN": "www.google.com.hk",
+        "CO": "www.google.com.co",
+        "CR": "www.google.co.cr",
+        "CU": "www.google.com.cu",
+        "CV": "www.google.cv",
+        "CY": "www.google.com.cy",
+        "CZ": "www.google.cz",
+        "DE": "www.google.de",
+        "DJ": "www.google.dj",
+        "DK": "www.google.dk",
+        "DM": "www.google.dm",
+        "DO": "www.google.com.do",
+        "DZ": "www.google.dz",
+        "EC": "www.google.com.ec",
+        "EE": "www.google.ee",
+        "EG": "www.google.com.eg",
+        "ES": "www.google.es",
+        "ET": "www.google.com.et",
+        "FI": "www.google.fi",
+        "FJ": "www.google.com.fj",
+        "FM": "www.google.fm",
+        "FR": "www.google.fr",
+        "GA": "www.google.ga",
+        "GE": "www.google.ge",
+        "GG": "www.google.gg",
+        "GH": "www.google.com.gh",
+        "GI": "www.google.com.gi",
+        "GL": "www.google.gl",
+        "GM": "www.google.gm",
+        "GR": "www.google.gr",
+        "GT": "www.google.com.gt",
+        "GY": "www.google.gy",
+        "HK": "www.google.com.hk",
+        "HN": "www.google.hn",
+        "HR": "www.google.hr",
+        "HT": "www.google.ht",
+        "HU": "www.google.hu",
+        "ID": "www.google.co.id",
+        "IE": "www.google.ie",
+        "IL": "www.google.co.il",
+        "IM": "www.google.im",
+        "IN": "www.google.co.in",
+        "IQ": "www.google.iq",
+        "IS": "www.google.is",
+        "IT": "www.google.it",
+        "JE": "www.google.je",
+        "JM": "www.google.com.jm",
+        "JO": "www.google.jo",
+        "JP": "www.google.co.jp",
+        "KE": "www.google.co.ke",
+        "KG": "www.google.kg",
+        "KH": "www.google.com.kh",
+        "KI": "www.google.ki",
+        "KR": "www.google.co.kr",
+        "KW": "www.google.com.kw",
+        "KZ": "www.google.kz",
+        "LA": "www.google.la",
+        "LB": "www.google.com.lb",
+        "LI": "www.google.li",
+        "LK": "www.google.lk",
+        "LS": "www.google.co.ls",
+        "LT": "www.google.lt",
+        "LU": "www.google.lu",
+        "LV": "www.google.lv",
+        "LY": "www.google.com.ly",
+        "MA": "www.google.co.ma",
+        "MD": "www.google.md",
+        "ME": "www.google.me",
+        "MG": "www.google.mg",
+        "MK": "www.google.mk",
+        "ML": "www.google.ml",
+        "MM": "www.google.com.mm",
+        "MN": "www.google.mn",
+        "MT": "www.google.com.mt",
+        "MU": "www.google.mu",
+        "MV": "www.google.mv",
+        "MW": "www.google.mw",
+        "MX": "www.google.com.mx",
+        "MY": "www.google.com.my",
+        "MZ": "www.google.co.mz",
+        "NA": "www.google.com.na",
+        "NE": "www.google.ne",
+        "NG": "www.google.com.ng",
+        "NI": "www.google.com.ni",
+        "NL": "www.google.nl",
+        "NO": "www.google.no",
+        "NP": "www.google.com.np",
+        "NR": "www.google.nr",
+        "NU": "www.google.nu",
+        "NZ": "www.google.co.nz",
+        "OM": "www.google.com.om",
+        "PA": "www.google.com.pa",
+        "PE": "www.google.com.pe",
+        "PG": "www.google.com.pg",
+        "PH": "www.google.com.ph",
+        "PK": "www.google.com.pk",
+        "PL": "www.google.pl",
+        "PN": "www.google.pn",
+        "PR": "www.google.com.pr",
+        "PS": "www.google.ps",
+        "PT": "www.google.pt",
+        "PY": "www.google.com.py",
+        "QA": "www.google.com.qa",
+        "RO": "www.google.ro",
+        "RS": "www.google.rs",
+        "RU": "www.google.ru",
+        "RW": "www.google.rw",
+        "SA": "www.google.com.sa",
+        "SB": "www.google.com.sb",
+        "SC": "www.google.sc",
+        "SE": "www.google.se",
+        "SG": "www.google.com.sg",
+        "SH": "www.google.sh",
+        "SI": "www.google.si",
+        "SK": "www.google.sk",
+        "SL": "www.google.com.sl",
+        "SM": "www.google.sm",
+        "SN": "www.google.sn",
+        "SO": "www.google.so",
+        "SR": "www.google.sr",
+        "ST": "www.google.st",
+        "SV": "www.google.com.sv",
+        "TD": "www.google.td",
+        "TG": "www.google.tg",
+        "TH": "www.google.co.th",
+        "TJ": "www.google.com.tj",
+        "TL": "www.google.tl",
+        "TM": "www.google.tm",
+        "TN": "www.google.tn",
+        "TO": "www.google.to",
+        "TR": "www.google.com.tr",
+        "TT": "www.google.tt",
+        "TW": "www.google.com.tw",
+        "TZ": "www.google.co.tz",
+        "UA": "www.google.com.ua",
+        "UG": "www.google.co.ug",
+        "UK": "www.google.co.uk",
+        "UY": "www.google.com.uy",
+        "UZ": "www.google.co.uz",
+        "VC": "www.google.com.vc",
+        "VE": "www.google.co.ve",
+        "VI": "www.google.co.vi",
+        "VN": "www.google.com.vn",
+        "VU": "www.google.vu",
+        "WS": "www.google.ws",
+        "ZA": "www.google.co.za",
+        "ZM": "www.google.co.zm",
+        "ZW": "www.google.co.zw"
+      }
+    },
+    "data_type": "traits_v1",
+    "languages": {
+      "af": "lang_af",
+      "ak": "lang_ak",
+      "am": "lang_am",
+      "ar": "lang_ar",
+      "az": "lang_az",
+      "be": "lang_be",
+      "bem": "lang_bem",
+      "bg": "lang_bg",
+      "bn": "lang_bn",
+      "br": "lang_br",
+      "bs": "lang_bs",
+      "ca": "lang_ca",
+      "ceb": "lang_ceb",
+      "chr": "lang_chr",
+      "ckb": "lang_ckb",
+      "co": "lang_co",
+      "cs": "lang_cs",
+      "cy": "lang_cy",
+      "da": "lang_da",
+      "de": "lang_de",
+      "ee": "lang_ee",
+      "el": "lang_el",
+      "en": "lang_en",
+      "eo": "lang_eo",
+      "es": "lang_es",
+      "et": "lang_et",
+      "eu": "lang_eu",
+      "fa": "lang_fa",
+      "fi": "lang_fi",
+      "fil": "lang_fil",
+      "fo": "lang_fo",
+      "fr": "lang_fr",
+      "fy": "lang_fy",
+      "ga": "lang_ga",
+      "gaa": "lang_gaa",
+      "gd": "lang_gd",
+      "gl": "lang_gl",
+      "gn": "lang_gn",
+      "gu": "lang_gu",
+      "ha": "lang_ha",
+      "haw": "lang_haw",
+      "he": "lang_iw",
+      "hi": "lang_hi",
+      "hr": "lang_hr",
+      "ht": "lang_ht",
+      "hu": "lang_hu",
+      "hy": "lang_hy",
+      "ia": "lang_ia",
+      "id": "lang_id",
+      "ig": "lang_ig",
+      "is": "lang_is",
+      "it": "lang_it",
+      "ja": "lang_ja",
+      "jv": "lang_jw",
+      "ka": "lang_ka",
+      "kk": "lang_kk",
+      "km": "lang_km",
+      "kn": "lang_kn",
+      "ko": "lang_ko",
+      "ky": "lang_ky",
+      "la": "lang_la",
+      "lg": "lang_lg",
+      "ln": "lang_ln",
+      "lo": "lang_lo",
+      "lt": "lang_lt",
+      "lv": "lang_lv",
+      "mfe": "lang_mfe",
+      "mg": "lang_mg",
+      "mi": "lang_mi",
+      "mk": "lang_mk",
+      "ml": "lang_ml",
+      "mn": "lang_mn",
+      "mr": "lang_mr",
+      "ms": "lang_ms",
+      "mt": "lang_mt",
+      "my": "lang_my",
+      "nb": "lang_no",
+      "ne": "lang_ne",
+      "nl": "lang_nl",
+      "nn": "lang_nn",
+      "nso": "lang_nso",
+      "ny": "lang_ny",
+      "nyn": "lang_nyn",
+      "oc": "lang_oc",
+      "om": "lang_om",
+      "or": "lang_or",
+      "pa": "lang_pa",
+      "pcm": "lang_pcm",
+      "pl": "lang_pl",
+      "ps": "lang_ps",
+      "pt": "lang_pt-BR",
+      "qu": "lang_qu",
+      "rm": "lang_rm",
+      "rn": "lang_rn",
+      "ro": "lang_ro",
+      "ru": "lang_ru",
+      "rw": "lang_rw",
+      "sd": "lang_sd",
+      "si": "lang_si",
+      "sk": "lang_sk",
+      "sl": "lang_sl",
+      "sn": "lang_sn",
+      "so": "lang_so",
+      "sq": "lang_sq",
+      "sr": "lang_sr",
+      "sr_Latn": "lang_sr-ME",
+      "st": "lang_st",
+      "su": "lang_su",
+      "sv": "lang_sv",
+      "sw": "lang_sw",
+      "ta": "lang_ta",
+      "te": "lang_te",
+      "tg": "lang_tg",
+      "th": "lang_th",
+      "ti": "lang_ti",
+      "tk": "lang_tk",
+      "tn": "lang_tn",
+      "to": "lang_to",
+      "tr": "lang_tr",
+      "tt": "lang_tt",
+      "ug": "lang_ug",
+      "uk": "lang_uk",
+      "ur": "lang_ur",
+      "uz": "lang_uz",
+      "vi": "lang_vi",
+      "wo": "lang_wo",
+      "xh": "lang_xh",
+      "yi": "lang_yi",
+      "yo": "lang_yo",
+      "zh": "lang_zh-CN",
+      "zh_Hans": "lang_zh-CN",
+      "zh_Hant": "lang_zh-TW",
+      "zu": "lang_zu"
+    },
+    "regions": {
+      "af-ZA": "ZA",
+      "ak-GH": "GH",
+      "ar-AE": "AE",
+      "ar-BH": "BH",
+      "ar-DJ": "DJ",
+      "ar-DZ": "DZ",
+      "ar-EG": "EG",
+      "ar-IL": "IL",
+      "ar-IQ": "IQ",
+      "ar-JO": "JO",
+      "ar-KW": "KW",
+      "ar-LB": "LB",
+      "ar-LY": "LY",
+      "ar-MA": "MA",
+      "ar-OM": "OM",
+      "ar-PS": "PS",
+      "ar-QA": "QA",
+      "ar-SA": "SA",
+      "ar-SO": "SO",
+      "ar-TD": "TD",
+      "ar-TN": "TN",
+      "be-BY": "BY",
+      "bg-BG": "BG",
+      "bn-IN": "IN",
+      "bs-BA": "BA",
+      "ca-AD": "AD",
+      "ca-ES": "ES",
+      "ceb-PH": "PH",
+      "ckb-IQ": "IQ",
+      "cs-CZ": "CZ",
+      "cy-GB": "GB",
+      "da-DK": "DK",
+      "de-AT": "AT",
+      "de-BE": "BE",
+      "de-CH": "CH",
+      "de-DE": "DE",
+      "de-LI": "LI",
+      "de-LU": "LU",
+      "ee-GH": "GH",
+      "el-CY": "CY",
+      "el-GR": "GR",
+      "en-AG": "AG",
+      "en-AS": "AS",
+      "en-AU": "AU",
+      "en-BI": "BI",
+      "en-BS": "BS",
+      "en-BW": "BW",
+      "en-BZ": "BZ",
+      "en-CA": "CA",
+      "en-CK": "CK",
+      "en-CM": "CM",
+      "en-DM": "DM",
+      "en-FJ": "FJ",
+      "en-FM": "FM",
+      "en-GB": "GB",
+      "en-GG": "GG",
+      "en-GH": "GH",
+      "en-GI": "GI",
+      "en-GM": "GM",
+      "en-GY": "GY",
+      "en-HK": "HK",
+      "en-IE": "IE",
+      "en-IM": "IM",
+      "en-IN": "IN",
+      "en-JE": "JE",
+      "en-JM": "JM",
+      "en-KE": "KE",
+      "en-KI": "KI",
+      "en-LS": "LS",
+      "en-MG": "MG",
+      "en-MT": "MT",
+      "en-MU": "MU",
+      "en-MW": "MW",
+      "en-NA": "NA",
+      "en-NG": "NG",
+      "en-NR": "NR",
+      "en-NU": "NU",
+      "en-NZ": "NZ",
+      "en-PG": "PG",
+      "en-PH": "PH",
+      "en-PK": "PK",
+      "en-PN": "PN",
+      "en-PR": "PR",
+      "en-RW": "RW",
+      "en-SB": "SB",
+      "en-SC": "SC",
+      "en-SG": "SG",
+      "en-SH": "SH",
+      "en-SL": "SL",
+      "en-TO": "TO",
+      "en-TT": "TT",
+      "en-TZ": "TZ",
+      "en-UG": "UG",
+      "en-US": "US",
+      "en-VC": "VC",
+      "en-VI": "VI",
+      "en-VU": "VU",
+      "en-WS": "WS",
+      "en-ZA": "ZA",
+      "en-ZM": "ZM",
+      "en-ZW": "ZW",
+      "es-AR": "AR",
+      "es-BO": "BO",
+      "es-CL": "CL",
+      "es-CO": "CO",
+      "es-CR": "CR",
+      "es-CU": "CU",
+      "es-DO": "DO",
+      "es-EC": "EC",
+      "es-ES": "ES",
+      "es-GT": "GT",
+      "es-HN": "HN",
+      "es-MX": "MX",
+      "es-NI": "NI",
+      "es-PA": "PA",
+      "es-PE": "PE",
+      "es-PR": "PR",
+      "es-PY": "PY",
+      "es-SV": "SV",
+      "es-US": "US",
+      "es-UY": "UY",
+      "es-VE": "VE",
+      "et-EE": "EE",
+      "eu-ES": "ES",
+      "fa-AF": "AF",
+      "fi-FI": "FI",
+      "fil-PH": "PH",
+      "fr-BE": "BE",
+      "fr-BF": "BF",
+      "fr-BI": "BI",
+      "fr-BJ": "BJ",
+      "fr-CA": "CA",
+      "fr-CD": "CD",
+      "fr-CF": "CF",
+      "fr-CG": "CG",
+      "fr-CH": "CH",
+      "fr-CI": "CI",
+      "fr-CM": "CM",
+      "fr-DJ": "DJ",
+      "fr-DZ": "DZ",
+      "fr-FR": "FR",
+      "fr-GA": "GA",
+      "fr-HT": "HT",
+      "fr-LU": "LU",
+      "fr-MA": "MA",
+      "fr-MG": "MG",
+      "fr-ML": "ML",
+      "fr-MU": "MU",
+      "fr-NE": "NE",
+      "fr-RW": "RW",
+      "fr-SC": "SC",
+      "fr-SN": "SN",
+      "fr-TD": "TD",
+      "fr-TG": "TG",
+      "fr-TN": "TN",
+      "fr-VU": "VU",
+      "fy-NL": "NL",
+      "ga-GB": "GB",
+      "ga-IE": "IE",
+      "gaa-GH": "GH",
+      "gd-GB": "GB",
+      "gl-ES": "ES",
+      "gn-PY": "PY",
+      "gu-IN": "IN",
+      "haw-US": "US",
+      "he-IL": "IL",
+      "hi-IN": "IN",
+      "hr-BA": "BA",
+      "hr-HR": "HR",
+      "ht-HT": "HT",
+      "hu-HU": "HU",
+      "hy-AM": "AM",
+      "id-ID": "ID",
+      "is-IS": "IS",
+      "it-CH": "CH",
+      "it-IT": "IT",
+      "it-SM": "SM",
+      "ja-JP": "JP",
+      "kk-KZ": "KZ",
+      "kn-IN": "IN",
+      "ko-KR": "KR",
+      "ky-KG": "KG",
+      "ln-CD": "CD",
+      "lt-LT": "LT",
+      "lv-LV": "LV",
+      "mg-MG": "MG",
+      "mi-NZ": "NZ",
+      "ml-IN": "IN",
+      "mr-IN": "IN",
+      "ms-SG": "SG",
+      "mt-MT": "MT",
+      "nb-NO": "NO",
+      "ne-IN": "IN",
+      "nl-BE": "BE",
+      "nl-NL": "NL",
+      "nl-SR": "SR",
+      "nn-NO": "NO",
+      "nso-ZA": "ZA",
+      "ny-MW": "MW",
+      "oc-ES": "ES",
+      "or-IN": "IN",
+      "pa-IN": "IN",
+      "pl-PL": "PL",
+      "ps-AF": "AF",
+      "pt-AO": "AO",
+      "pt-BR": "BR",
+      "pt-CV": "CV",
+      "pt-MZ": "MZ",
+      "pt-PT": "PT",
+      "pt-ST": "ST",
+      "pt-TL": "TL",
+      "qu-BO": "BO",
+      "qu-EC": "EC",
+      "qu-PE": "PE",
+      "rm-CH": "CH",
+      "rn-BI": "BI",
+      "ro-MD": "MD",
+      "ro-RO": "RO",
+      "ru-BY": "BY",
+      "ru-KG": "KG",
+      "ru-KZ": "KZ",
+      "ru-RU": "RU",
+      "ru-UA": "UA",
+      "rw-RW": "RW",
+      "sd-IN": "IN",
+      "sk-SK": "SK",
+      "sl-SI": "SI",
+      "sn-ZW": "ZW",
+      "so-SO": "SO",
+      "sr-BA": "BA",
+      "sr-RS": "RS",
+      "st-LS": "LS",
+      "st-ZA": "ZA",
+      "sv-FI": "FI",
+      "sv-SE": "SE",
+      "sw-CD": "CD",
+      "sw-KE": "KE",
+      "sw-TZ": "TZ",
+      "sw-UG": "UG",
+      "ta-IN": "IN",
+      "ta-SG": "SG",
+      "te-IN": "IN",
+      "th-TH": "TH",
+      "tn-BW": "BW",
+      "tn-ZA": "ZA",
+      "to-TO": "TO",
+      "tr-CY": "CY",
+      "tr-TR": "TR",
+      "tt-RU": "RU",
+      "uk-UA": "UA",
+      "ur-IN": "IN",
+      "ur-PK": "PK",
+      "vi-VN": "VN",
+      "wo-SN": "SN",
+      "xh-ZA": "ZA",
+      "yo-NG": "NG",
+      "zh-CN": "HK",
+      "zh-HK": "HK",
+      "zh-SG": "SG",
+      "zh-TW": "TW",
+      "zu-ZA": "ZA"
+    }
+  },
   "google images": {
     "all_locale": "ZZ",
     "custom": {

searx/engines/google.py

@@ -32,7 +32,6 @@ from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
     extract_text,
-    gen_gsa_useragent,
 )
 
 if t.TYPE_CHECKING:
@@ -197,7 +196,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     #   https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
 
     # https://github.com/searxng/searxng/issues/2515#issuecomment-1607150817
-    ret_val["params"]["hl"] = f"{lang_code}-{country}"
+    ret_val["params"]["hl"] = f"{lang_code}"
 
     # lr parameter:
     #   The lr (language restrict) parameter restricts search results to
@@ -268,7 +267,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     # HTTP headers
 
     ret_val["headers"]["Accept"] = "*/*"
-    ret_val["headers"]["User-Agent"] = gen_gsa_useragent()
 
     # Cookies
 

searx/engines/google_cse.py

@@ -0,0 +1,158 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Google Custom Search Engine"""
+
+import datetime
+import typing as t
+from json import loads
+from urllib.parse import urlencode
+
+from searx.enginelib import EngineCache
+from searx.exceptions import SearxEngineAPIException, SearxEngineTooManyRequestsException
+from searx.network import get
+from searx.result_types import EngineResults
+
+from searx.engines.google import fetch_traits  # pylint: disable=unused-import
+from searx.engines.google import filter_mapping, get_google_info
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
+
+about = {
+    "website": "https://www.google.com",
+    "wikidata_id": "Q2233943",
+    "official_api_documentation": "https://developers.google.com/custom-search/docs/element",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "JSONP",
+    "description": "Platform for creating custom search engines based on Google Search.",
+}
+
+categories = ["general", "web"]
+paging = True
+max_page = 5
+page_size = 20
+time_range_support = True
+language_support = True
+safesearch = True
+
+CX = "partner-pub-8993703457585266:4862972284"  # blackle.com
+
+CACHE: EngineCache
+
+
+def setup(engine_settings: dict[str, t.Any]) -> bool:
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])
+    return True
+
+
+def _cse_token() -> dict[str, str]:
+    token: dict[str, str] = CACHE.get(CX)
+    if token:
+        return token
+
+    resp = get(f"https://www.google.com/cse/cse.js?cx={CX}", timeout=10)
+    if not resp.ok:
+        raise SearxEngineAPIException("failed to obtain cse token")
+
+    end = resp.text.rfind("});")
+    start = resp.text.rfind("({")
+    opts: dict[str, str] = loads(resp.text[start + 1 : end + 1])
+
+    cse_tok = opts.get("cse_token")
+    if not cse_tok:
+        raise SearxEngineAPIException("failed to obtain cse token")
+
+    exp = opts.get("exp")
+    token = {
+        "cse_tok": cse_tok,
+        "cselibv": opts.get("cselibVersion", ""),
+        "exp": ",".join(exp) if exp else "",
+    }
+    CACHE.set(CX, token, expire=3600)
+    return token
+
+
+def _get_start_and_end_date_str(time_range: str) -> tuple[str, str]:
+    time_range_map = {"day": 1, "week": 7, "month": 30, "year": 365}
+
+    end_date = datetime.datetime.now()
+    start_date = end_date - datetime.timedelta(days=time_range_map[time_range])
+
+    return start_date.strftime("%Y%m%d"), end_date.strftime("%Y%m%d")
+
+
+def request(query: str, params: "OnlineParams") -> None:
+    token = _cse_token()
+
+    google_info = get_google_info(params, traits)
+    info: dict[str, str] = google_info["params"]
+
+    hl = info["hl"]
+    if params.get("searxng_locale", "all") == "all" or "ZZ" in hl:
+        hl = "en"
+
+    args = {
+        "rsz": "filtered_cse",
+        "num": str(page_size),
+        "hl": hl,
+        "cselibv": token["cselibv"],
+        "cx": CX,
+        "q": query,
+        "safe": filter_mapping[params["safesearch"]],
+        "cse_tok": token["cse_tok"],
+        "callback": "_",
+        "rurl": "",
+    }
+    if params["time_range"]:
+        start_date, end_date = _get_start_and_end_date_str(params["time_range"])
+        args["sort"] = f"date:r:{start_date}:{end_date}"
+
+    if info.get("lr"):
+        args["lr"] = info["lr"]
+    if info.get("cr"):
+        args["cr"] = info["cr"]
+    if google_info["country"] not in (None, "ZZ"):
+        args["gl"] = google_info["country"]
+    if token["exp"]:
+        args["exp"] = token["exp"]
+
+    start = (params["pageno"] - 1) * page_size
+    if start:
+        args["start"] = str(start)
+
+    params["url"] = "https://cse.google.com/cse/element/v1?" + urlencode(args)
+    params["cookies"] = google_info["cookies"]
+    params["headers"].update(google_info["headers"])
+    params["headers"]["Referer"] = "https://cse.google.com/"
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    json_resp = resp.text[resp.text.find("{") : resp.text.rfind("}") + 1]
+    data = loads(json_resp)
+
+    # not the real types, but a sufficient approximation
+    item: dict[str, str]
+    error: dict[str, str | int]
+
+    if error := data.get("error"):
+        message = error.get("message", "unknown error")
+        if error.get("code") == 429:
+            raise SearxEngineTooManyRequestsException(message=f"google cse: {message}")
+        raise SearxEngineAPIException(f"google cse: {message}")
+
+    results = EngineResults()
+    for item in data.get("results", []):
+        url = item.get("unescapedUrl")
+        if not url:
+            continue
+        results.add(
+            results.types.MainResult(
+                url=url,
+                title=item.get("titleNoFormatting", ""),
+                content=item.get("contentNoFormatting", ""),
+                thumbnail=item.get("richSnippet", {}).get("cseThumbnail", {}).get("src", ""),  # type: ignore
+            )
+        )
+    return results

searx/engines/neocities.py

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Neocities_ is open source software for creating blogs.
+
+.. _Neocities : https://github.com/neocities/neocities
+"""
+
+from urllib.parse import urlencode
+import typing as t
+
+from lxml import html
+
+from searx.utils import eval_xpath, eval_xpath_list, extract_text
+from searx.result_types import EngineResults
+
+if t.TYPE_CHECKING:
+    from extended_types import SXNG_Response
+    from search.processors import OnlineParams
+
+
+# Engine metadata
+about = {
+    "website": "https://neocities.org/",
+    "wikidata_id": "Q17071099",
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+}
+
+# Engine configuration
+categories = ["general", "blogs"]
+paging = True
+
+# Search URL
+base_url = "https://neocities.org"
+
+results_xpath = "//div[@class='result-item']"
+url_xpath = './/div[@class="result-url"]/a/@href'
+title_xpath = './/h3[@class="result-title"]/a/text()'
+content_xpath = './/p[@class="result-snippet"]//text()'
+screenshot_xpath = './/a[@class="result-screenshot"]/img/@src'
+
+
+def request(query: str, params: "OnlineParams") -> None:
+    query_params: dict[str, t.Any] = {"q": query}
+    if params['pageno'] > 1:
+        offset = (params["pageno"] - 1) * 100
+        query_params["start"] = offset
+    params["url"] = f"{base_url}/search?{urlencode(query_params)}"
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    results = EngineResults()
+    dom = html.fromstring(resp.text)
+
+    for result in eval_xpath_list(dom, results_xpath):
+        results.add(
+            results.types.MainResult(
+                url=extract_text(eval_xpath(result, url_xpath)),
+                title=extract_text(eval_xpath(result, title_xpath)) or "",
+                content=extract_text(eval_xpath(result, content_xpath)) or "",
+                thumbnail=base_url + (extract_text(eval_xpath(result, screenshot_xpath)) or ""),
+            )
+        )
+
+    return results

searx/settings.yml

@@ -1199,6 +1199,11 @@ engines:
   - name: google
     engine: google
     shortcut: go
+    inactive: true
+
+  - name: google cse
+    engine: google_cse
+    shortcut: goc
 
   - name: google images
     engine: google_images
@@ -1211,6 +1216,7 @@ engines:
   - name: google videos
     engine: google_videos
     shortcut: gov
+    inactive: true
 
   - name: google scholar
     engine: google_scholar
@@ -1691,6 +1697,12 @@ engines:
     shortcut: mwm
     disabled: true
 
+  - name: neocities
+    engine: neocities
+    shortcut: nc
+    disabled: true
+    inactive: true
+
   - name: niconico
     engine: niconico
     shortcut: nico

searx/utils.py

@@ -26,7 +26,7 @@ from lxml.etree import XPath, XPathError, XPathSyntaxError
 from lxml.etree import ElementBase, _Element  # pyright: ignore[reportPrivateUsage]
 
 from searx import settings
-from searx.data import USER_AGENTS, gsa_useragents_loader
+from searx.data import USER_AGENTS
 from searx.version import VERSION_TAG
 from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
 from searx import logger
@@ -82,14 +82,6 @@ def gen_useragent(os_string: str | None = None) -> str:
     )
 
 
-def gen_gsa_useragent() -> str:
-    """Return a random "Google Go App" User Agent suitable for Google
-
-    See searx/data/gsa_useragents.txt
-    """
-    return choice(gsa_useragents_loader()) + " NSTNWV"
-
-
 class HTMLTextExtractor(HTMLParser):
     """Internal class to extract text from HTML"""
 

searxng_extra/update/update_engine_traits.py

@@ -131,8 +131,8 @@ def fetch_traits_map() -> EngineTraitsMap:
 def filter_locales(traits_map: EngineTraitsMap) -> set[str]:
     """Filter language & region tags by a threshold."""
 
-    min_eng_per_region = 18
+    min_eng_per_region = 19
-    min_eng_per_lang = 23
+    min_eng_per_lang = 24
 
     _: dict[str, int] = {}
     for eng in traits_map.values():

searxng_extra/update/update_gsa_useragents.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""This script fetches user agents suitable for Google.
-
-Output file: :origin:`searx/data/gsa_useragents.txt` (:origin:`CI Update data
-...  <.github/workflows/data-update.yml>`).
-
-.. Source for user agents: https://github.com/intoli/user-agents/
-
-"""
-
-from gzip import decompress
-from json import loads
-
-from searx.data import data_dir
-from searx.network import get as http_get
-from searx.utils import searxng_useragent
-
-DATA_FILE = data_dir / "gsa_useragents.txt"
-URL = "https://raw.githubusercontent.com/intoli/user-agents/main/src/user-agents.json.gz"
-
-
-def fetch_gsa_useragents() -> list[str]:
-    response = http_get(URL, timeout=3.0, headers={"User-Agent": searxng_useragent()})
-    response.raise_for_status()
-
-    suas: set[str] = set()
-    for ua in loads(decompress(response.content)):
-        if (
-            "Android" in ua["userAgent"]
-            and "Chrome" in ua["userAgent"]
-            and "Samsung" not in ua["userAgent"]
-            and "Android 10; K" not in ua["userAgent"]
-        ):
-            suas.add(ua["userAgent"])
-
-    luas = list(suas)
-    luas.sort()
-
-    return luas
-
-
-if __name__ == "__main__":
-    useragents = fetch_gsa_useragents()
-    with DATA_FILE.open("w", encoding="utf-8") as f:
-        f.write("\n".join(useragents))

utils/lib_sxng_data.sh

@@ -7,7 +7,6 @@ data.:
   all           : update searx/sxng_locales.py and searx/data/*
   traits        : update searx/data/engine_traits.json & searx/sxng_locales.py
   useragents    : update searx/data/useragents.json with the most recent versions of Firefox
-  gsa_useragents: update searx/data/gsa_useragents.txt with compatible useragents
   locales       : update searx/data/locales.json from babel
   currencies    : update searx/data/currencies.json from wikidata
 EOF
@@ -20,7 +19,6 @@ data.all() {
         pyenv.activate
         data.traits
         data.useragents
-        data.gsa_useragents
         data.locales
 
         build_msg DATA "update searx/data/osm_keys_tags.json"
@@ -55,12 +53,6 @@ data.useragents() {
     dump_return $?
 }
 
-data.gsa_useragents() {
-    build_msg DATA "update searx/data/gsa_useragents.txt"
-    pyenv.cmd python searxng_extra/update/update_gsa_useragents.py
-    dump_return $?
-}
-
 data.locales() {
     (
         set -e