mirror of
https://github.com/searxng/searxng.git
synced 2026-07-17 21:41:24 +00:00
Compare commits
4 Commits
747cec4c23
...
a6438586a5
| Author | SHA1 | Date | |
|---|---|---|---|
|
a6438586a5 | ||
|
fd5eb84a37 | ||
|
888364c1ce | ||
|
|
1cdf01a719 |
1
.github/workflows/data-update.yml
vendored
1
.github/workflows/data-update.yml
vendored
@@ -33,7 +33,6 @@ jobs:
|
||||
- update_engine_traits.py
|
||||
- update_wikidata_units.py
|
||||
- update_engine_descriptions.py
|
||||
- update_gsa_useragents.py
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
2
Makefile
2
Makefile
@@ -63,7 +63,7 @@ format: format.python format.shell
|
||||
# wrap ./manage script
|
||||
|
||||
MANAGE += weblate.translations.commit weblate.push.translations
|
||||
MANAGE += data.all data.traits data.useragents data.gsa_useragents data.locales data.currencies
|
||||
MANAGE += data.all data.traits data.useragents data.locales data.currencies
|
||||
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
|
||||
MANAGE += podman.build
|
||||
MANAGE += docker.build docker.buildx
|
||||
|
||||
@@ -6,7 +6,7 @@ make data.all
|
||||
"""
|
||||
# pylint: disable=invalid-name
|
||||
|
||||
__all__ = ["ahmia_blacklist_loader", "gsa_useragents_loader", "data_dir", "get_cache"]
|
||||
__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
|
||||
|
||||
import json
|
||||
import typing as t
|
||||
@@ -63,7 +63,6 @@ lazy_globals = {
|
||||
"ENGINE_TRAITS": None,
|
||||
"LOCALES": None,
|
||||
"TRACKER_PATTERNS": TrackerPatternsDB(),
|
||||
"GSA_USER_AGENTS": None,
|
||||
}
|
||||
|
||||
data_json_files = {
|
||||
@@ -106,24 +105,3 @@ def ahmia_blacklist_loader() -> list[str]:
|
||||
"""
|
||||
with open(data_dir / 'ahmia_blacklist.txt', encoding='utf-8') as f:
|
||||
return f.read().split()
|
||||
|
||||
|
||||
def gsa_useragents_loader() -> list[str]:
|
||||
"""Load data from `gsa_useragents.txt` and return a list of user agents
|
||||
suitable for Google. The user agents are fetched by::
|
||||
|
||||
searxng_extra/update/update_gsa_useragents.py
|
||||
|
||||
This function is used by :py:mod:`searx.engines.google`.
|
||||
|
||||
"""
|
||||
data = lazy_globals["GSA_USER_AGENTS"]
|
||||
if data is not None:
|
||||
return data
|
||||
|
||||
log.debug("init searx.data.%s", "GSA_USER_AGENTS")
|
||||
|
||||
with open(data_dir / 'gsa_useragents.txt', encoding='utf-8') as f:
|
||||
lazy_globals["GSA_USER_AGENTS"] = f.read().splitlines()
|
||||
|
||||
return lazy_globals["GSA_USER_AGENTS"]
|
||||
|
||||
@@ -3476,6 +3476,598 @@
|
||||
"zu-ZA": "ZA"
|
||||
}
|
||||
},
|
||||
"google cse": {
|
||||
"all_locale": "ZZ",
|
||||
"custom": {
|
||||
"supported_domains": {
|
||||
"AD": "www.google.ad",
|
||||
"AE": "www.google.ae",
|
||||
"AF": "www.google.com.af",
|
||||
"AG": "www.google.com.ag",
|
||||
"AL": "www.google.al",
|
||||
"AM": "www.google.am",
|
||||
"AO": "www.google.co.ao",
|
||||
"AR": "www.google.com.ar",
|
||||
"AS": "www.google.as",
|
||||
"AT": "www.google.at",
|
||||
"AU": "www.google.com.au",
|
||||
"AZ": "www.google.az",
|
||||
"BA": "www.google.ba",
|
||||
"BD": "www.google.com.bd",
|
||||
"BE": "www.google.be",
|
||||
"BF": "www.google.bf",
|
||||
"BG": "www.google.bg",
|
||||
"BH": "www.google.com.bh",
|
||||
"BI": "www.google.bi",
|
||||
"BJ": "www.google.bj",
|
||||
"BN": "www.google.com.bn",
|
||||
"BO": "www.google.com.bo",
|
||||
"BR": "www.google.com.br",
|
||||
"BS": "www.google.bs",
|
||||
"BT": "www.google.bt",
|
||||
"BW": "www.google.co.bw",
|
||||
"BY": "www.google.by",
|
||||
"BZ": "www.google.com.bz",
|
||||
"CA": "www.google.ca",
|
||||
"CAT": "www.google.cat",
|
||||
"CD": "www.google.cd",
|
||||
"CF": "www.google.cf",
|
||||
"CG": "www.google.cg",
|
||||
"CH": "www.google.ch",
|
||||
"CI": "www.google.ci",
|
||||
"CK": "www.google.co.ck",
|
||||
"CL": "www.google.cl",
|
||||
"CM": "www.google.cm",
|
||||
"CN": "www.google.com.hk",
|
||||
"CO": "www.google.com.co",
|
||||
"CR": "www.google.co.cr",
|
||||
"CU": "www.google.com.cu",
|
||||
"CV": "www.google.cv",
|
||||
"CY": "www.google.com.cy",
|
||||
"CZ": "www.google.cz",
|
||||
"DE": "www.google.de",
|
||||
"DJ": "www.google.dj",
|
||||
"DK": "www.google.dk",
|
||||
"DM": "www.google.dm",
|
||||
"DO": "www.google.com.do",
|
||||
"DZ": "www.google.dz",
|
||||
"EC": "www.google.com.ec",
|
||||
"EE": "www.google.ee",
|
||||
"EG": "www.google.com.eg",
|
||||
"ES": "www.google.es",
|
||||
"ET": "www.google.com.et",
|
||||
"FI": "www.google.fi",
|
||||
"FJ": "www.google.com.fj",
|
||||
"FM": "www.google.fm",
|
||||
"FR": "www.google.fr",
|
||||
"GA": "www.google.ga",
|
||||
"GE": "www.google.ge",
|
||||
"GG": "www.google.gg",
|
||||
"GH": "www.google.com.gh",
|
||||
"GI": "www.google.com.gi",
|
||||
"GL": "www.google.gl",
|
||||
"GM": "www.google.gm",
|
||||
"GR": "www.google.gr",
|
||||
"GT": "www.google.com.gt",
|
||||
"GY": "www.google.gy",
|
||||
"HK": "www.google.com.hk",
|
||||
"HN": "www.google.hn",
|
||||
"HR": "www.google.hr",
|
||||
"HT": "www.google.ht",
|
||||
"HU": "www.google.hu",
|
||||
"ID": "www.google.co.id",
|
||||
"IE": "www.google.ie",
|
||||
"IL": "www.google.co.il",
|
||||
"IM": "www.google.im",
|
||||
"IN": "www.google.co.in",
|
||||
"IQ": "www.google.iq",
|
||||
"IS": "www.google.is",
|
||||
"IT": "www.google.it",
|
||||
"JE": "www.google.je",
|
||||
"JM": "www.google.com.jm",
|
||||
"JO": "www.google.jo",
|
||||
"JP": "www.google.co.jp",
|
||||
"KE": "www.google.co.ke",
|
||||
"KG": "www.google.kg",
|
||||
"KH": "www.google.com.kh",
|
||||
"KI": "www.google.ki",
|
||||
"KR": "www.google.co.kr",
|
||||
"KW": "www.google.com.kw",
|
||||
"KZ": "www.google.kz",
|
||||
"LA": "www.google.la",
|
||||
"LB": "www.google.com.lb",
|
||||
"LI": "www.google.li",
|
||||
"LK": "www.google.lk",
|
||||
"LS": "www.google.co.ls",
|
||||
"LT": "www.google.lt",
|
||||
"LU": "www.google.lu",
|
||||
"LV": "www.google.lv",
|
||||
"LY": "www.google.com.ly",
|
||||
"MA": "www.google.co.ma",
|
||||
"MD": "www.google.md",
|
||||
"ME": "www.google.me",
|
||||
"MG": "www.google.mg",
|
||||
"MK": "www.google.mk",
|
||||
"ML": "www.google.ml",
|
||||
"MM": "www.google.com.mm",
|
||||
"MN": "www.google.mn",
|
||||
"MT": "www.google.com.mt",
|
||||
"MU": "www.google.mu",
|
||||
"MV": "www.google.mv",
|
||||
"MW": "www.google.mw",
|
||||
"MX": "www.google.com.mx",
|
||||
"MY": "www.google.com.my",
|
||||
"MZ": "www.google.co.mz",
|
||||
"NA": "www.google.com.na",
|
||||
"NE": "www.google.ne",
|
||||
"NG": "www.google.com.ng",
|
||||
"NI": "www.google.com.ni",
|
||||
"NL": "www.google.nl",
|
||||
"NO": "www.google.no",
|
||||
"NP": "www.google.com.np",
|
||||
"NR": "www.google.nr",
|
||||
"NU": "www.google.nu",
|
||||
"NZ": "www.google.co.nz",
|
||||
"OM": "www.google.com.om",
|
||||
"PA": "www.google.com.pa",
|
||||
"PE": "www.google.com.pe",
|
||||
"PG": "www.google.com.pg",
|
||||
"PH": "www.google.com.ph",
|
||||
"PK": "www.google.com.pk",
|
||||
"PL": "www.google.pl",
|
||||
"PN": "www.google.pn",
|
||||
"PR": "www.google.com.pr",
|
||||
"PS": "www.google.ps",
|
||||
"PT": "www.google.pt",
|
||||
"PY": "www.google.com.py",
|
||||
"QA": "www.google.com.qa",
|
||||
"RO": "www.google.ro",
|
||||
"RS": "www.google.rs",
|
||||
"RU": "www.google.ru",
|
||||
"RW": "www.google.rw",
|
||||
"SA": "www.google.com.sa",
|
||||
"SB": "www.google.com.sb",
|
||||
"SC": "www.google.sc",
|
||||
"SE": "www.google.se",
|
||||
"SG": "www.google.com.sg",
|
||||
"SH": "www.google.sh",
|
||||
"SI": "www.google.si",
|
||||
"SK": "www.google.sk",
|
||||
"SL": "www.google.com.sl",
|
||||
"SM": "www.google.sm",
|
||||
"SN": "www.google.sn",
|
||||
"SO": "www.google.so",
|
||||
"SR": "www.google.sr",
|
||||
"ST": "www.google.st",
|
||||
"SV": "www.google.com.sv",
|
||||
"TD": "www.google.td",
|
||||
"TG": "www.google.tg",
|
||||
"TH": "www.google.co.th",
|
||||
"TJ": "www.google.com.tj",
|
||||
"TL": "www.google.tl",
|
||||
"TM": "www.google.tm",
|
||||
"TN": "www.google.tn",
|
||||
"TO": "www.google.to",
|
||||
"TR": "www.google.com.tr",
|
||||
"TT": "www.google.tt",
|
||||
"TW": "www.google.com.tw",
|
||||
"TZ": "www.google.co.tz",
|
||||
"UA": "www.google.com.ua",
|
||||
"UG": "www.google.co.ug",
|
||||
"UK": "www.google.co.uk",
|
||||
"UY": "www.google.com.uy",
|
||||
"UZ": "www.google.co.uz",
|
||||
"VC": "www.google.com.vc",
|
||||
"VE": "www.google.co.ve",
|
||||
"VI": "www.google.co.vi",
|
||||
"VN": "www.google.com.vn",
|
||||
"VU": "www.google.vu",
|
||||
"WS": "www.google.ws",
|
||||
"ZA": "www.google.co.za",
|
||||
"ZM": "www.google.co.zm",
|
||||
"ZW": "www.google.co.zw"
|
||||
}
|
||||
},
|
||||
"data_type": "traits_v1",
|
||||
"languages": {
|
||||
"af": "lang_af",
|
||||
"ak": "lang_ak",
|
||||
"am": "lang_am",
|
||||
"ar": "lang_ar",
|
||||
"az": "lang_az",
|
||||
"be": "lang_be",
|
||||
"bem": "lang_bem",
|
||||
"bg": "lang_bg",
|
||||
"bn": "lang_bn",
|
||||
"br": "lang_br",
|
||||
"bs": "lang_bs",
|
||||
"ca": "lang_ca",
|
||||
"ceb": "lang_ceb",
|
||||
"chr": "lang_chr",
|
||||
"ckb": "lang_ckb",
|
||||
"co": "lang_co",
|
||||
"cs": "lang_cs",
|
||||
"cy": "lang_cy",
|
||||
"da": "lang_da",
|
||||
"de": "lang_de",
|
||||
"ee": "lang_ee",
|
||||
"el": "lang_el",
|
||||
"en": "lang_en",
|
||||
"eo": "lang_eo",
|
||||
"es": "lang_es",
|
||||
"et": "lang_et",
|
||||
"eu": "lang_eu",
|
||||
"fa": "lang_fa",
|
||||
"fi": "lang_fi",
|
||||
"fil": "lang_fil",
|
||||
"fo": "lang_fo",
|
||||
"fr": "lang_fr",
|
||||
"fy": "lang_fy",
|
||||
"ga": "lang_ga",
|
||||
"gaa": "lang_gaa",
|
||||
"gd": "lang_gd",
|
||||
"gl": "lang_gl",
|
||||
"gn": "lang_gn",
|
||||
"gu": "lang_gu",
|
||||
"ha": "lang_ha",
|
||||
"haw": "lang_haw",
|
||||
"he": "lang_iw",
|
||||
"hi": "lang_hi",
|
||||
"hr": "lang_hr",
|
||||
"ht": "lang_ht",
|
||||
"hu": "lang_hu",
|
||||
"hy": "lang_hy",
|
||||
"ia": "lang_ia",
|
||||
"id": "lang_id",
|
||||
"ig": "lang_ig",
|
||||
"is": "lang_is",
|
||||
"it": "lang_it",
|
||||
"ja": "lang_ja",
|
||||
"jv": "lang_jw",
|
||||
"ka": "lang_ka",
|
||||
"kk": "lang_kk",
|
||||
"km": "lang_km",
|
||||
"kn": "lang_kn",
|
||||
"ko": "lang_ko",
|
||||
"ky": "lang_ky",
|
||||
"la": "lang_la",
|
||||
"lg": "lang_lg",
|
||||
"ln": "lang_ln",
|
||||
"lo": "lang_lo",
|
||||
"lt": "lang_lt",
|
||||
"lv": "lang_lv",
|
||||
"mfe": "lang_mfe",
|
||||
"mg": "lang_mg",
|
||||
"mi": "lang_mi",
|
||||
"mk": "lang_mk",
|
||||
"ml": "lang_ml",
|
||||
"mn": "lang_mn",
|
||||
"mr": "lang_mr",
|
||||
"ms": "lang_ms",
|
||||
"mt": "lang_mt",
|
||||
"my": "lang_my",
|
||||
"nb": "lang_no",
|
||||
"ne": "lang_ne",
|
||||
"nl": "lang_nl",
|
||||
"nn": "lang_nn",
|
||||
"nso": "lang_nso",
|
||||
"ny": "lang_ny",
|
||||
"nyn": "lang_nyn",
|
||||
"oc": "lang_oc",
|
||||
"om": "lang_om",
|
||||
"or": "lang_or",
|
||||
"pa": "lang_pa",
|
||||
"pcm": "lang_pcm",
|
||||
"pl": "lang_pl",
|
||||
"ps": "lang_ps",
|
||||
"pt": "lang_pt-BR",
|
||||
"qu": "lang_qu",
|
||||
"rm": "lang_rm",
|
||||
"rn": "lang_rn",
|
||||
"ro": "lang_ro",
|
||||
"ru": "lang_ru",
|
||||
"rw": "lang_rw",
|
||||
"sd": "lang_sd",
|
||||
"si": "lang_si",
|
||||
"sk": "lang_sk",
|
||||
"sl": "lang_sl",
|
||||
"sn": "lang_sn",
|
||||
"so": "lang_so",
|
||||
"sq": "lang_sq",
|
||||
"sr": "lang_sr",
|
||||
"sr_Latn": "lang_sr-ME",
|
||||
"st": "lang_st",
|
||||
"su": "lang_su",
|
||||
"sv": "lang_sv",
|
||||
"sw": "lang_sw",
|
||||
"ta": "lang_ta",
|
||||
"te": "lang_te",
|
||||
"tg": "lang_tg",
|
||||
"th": "lang_th",
|
||||
"ti": "lang_ti",
|
||||
"tk": "lang_tk",
|
||||
"tn": "lang_tn",
|
||||
"to": "lang_to",
|
||||
"tr": "lang_tr",
|
||||
"tt": "lang_tt",
|
||||
"ug": "lang_ug",
|
||||
"uk": "lang_uk",
|
||||
"ur": "lang_ur",
|
||||
"uz": "lang_uz",
|
||||
"vi": "lang_vi",
|
||||
"wo": "lang_wo",
|
||||
"xh": "lang_xh",
|
||||
"yi": "lang_yi",
|
||||
"yo": "lang_yo",
|
||||
"zh": "lang_zh-CN",
|
||||
"zh_Hans": "lang_zh-CN",
|
||||
"zh_Hant": "lang_zh-TW",
|
||||
"zu": "lang_zu"
|
||||
},
|
||||
"regions": {
|
||||
"af-ZA": "ZA",
|
||||
"ak-GH": "GH",
|
||||
"ar-AE": "AE",
|
||||
"ar-BH": "BH",
|
||||
"ar-DJ": "DJ",
|
||||
"ar-DZ": "DZ",
|
||||
"ar-EG": "EG",
|
||||
"ar-IL": "IL",
|
||||
"ar-IQ": "IQ",
|
||||
"ar-JO": "JO",
|
||||
"ar-KW": "KW",
|
||||
"ar-LB": "LB",
|
||||
"ar-LY": "LY",
|
||||
"ar-MA": "MA",
|
||||
"ar-OM": "OM",
|
||||
"ar-PS": "PS",
|
||||
"ar-QA": "QA",
|
||||
"ar-SA": "SA",
|
||||
"ar-SO": "SO",
|
||||
"ar-TD": "TD",
|
||||
"ar-TN": "TN",
|
||||
"be-BY": "BY",
|
||||
"bg-BG": "BG",
|
||||
"bn-IN": "IN",
|
||||
"bs-BA": "BA",
|
||||
"ca-AD": "AD",
|
||||
"ca-ES": "ES",
|
||||
"ceb-PH": "PH",
|
||||
"ckb-IQ": "IQ",
|
||||
"cs-CZ": "CZ",
|
||||
"cy-GB": "GB",
|
||||
"da-DK": "DK",
|
||||
"de-AT": "AT",
|
||||
"de-BE": "BE",
|
||||
"de-CH": "CH",
|
||||
"de-DE": "DE",
|
||||
"de-LI": "LI",
|
||||
"de-LU": "LU",
|
||||
"ee-GH": "GH",
|
||||
"el-CY": "CY",
|
||||
"el-GR": "GR",
|
||||
"en-AG": "AG",
|
||||
"en-AS": "AS",
|
||||
"en-AU": "AU",
|
||||
"en-BI": "BI",
|
||||
"en-BS": "BS",
|
||||
"en-BW": "BW",
|
||||
"en-BZ": "BZ",
|
||||
"en-CA": "CA",
|
||||
"en-CK": "CK",
|
||||
"en-CM": "CM",
|
||||
"en-DM": "DM",
|
||||
"en-FJ": "FJ",
|
||||
"en-FM": "FM",
|
||||
"en-GB": "GB",
|
||||
"en-GG": "GG",
|
||||
"en-GH": "GH",
|
||||
"en-GI": "GI",
|
||||
"en-GM": "GM",
|
||||
"en-GY": "GY",
|
||||
"en-HK": "HK",
|
||||
"en-IE": "IE",
|
||||
"en-IM": "IM",
|
||||
"en-IN": "IN",
|
||||
"en-JE": "JE",
|
||||
"en-JM": "JM",
|
||||
"en-KE": "KE",
|
||||
"en-KI": "KI",
|
||||
"en-LS": "LS",
|
||||
"en-MG": "MG",
|
||||
"en-MT": "MT",
|
||||
"en-MU": "MU",
|
||||
"en-MW": "MW",
|
||||
"en-NA": "NA",
|
||||
"en-NG": "NG",
|
||||
"en-NR": "NR",
|
||||
"en-NU": "NU",
|
||||
"en-NZ": "NZ",
|
||||
"en-PG": "PG",
|
||||
"en-PH": "PH",
|
||||
"en-PK": "PK",
|
||||
"en-PN": "PN",
|
||||
"en-PR": "PR",
|
||||
"en-RW": "RW",
|
||||
"en-SB": "SB",
|
||||
"en-SC": "SC",
|
||||
"en-SG": "SG",
|
||||
"en-SH": "SH",
|
||||
"en-SL": "SL",
|
||||
"en-TO": "TO",
|
||||
"en-TT": "TT",
|
||||
"en-TZ": "TZ",
|
||||
"en-UG": "UG",
|
||||
"en-US": "US",
|
||||
"en-VC": "VC",
|
||||
"en-VI": "VI",
|
||||
"en-VU": "VU",
|
||||
"en-WS": "WS",
|
||||
"en-ZA": "ZA",
|
||||
"en-ZM": "ZM",
|
||||
"en-ZW": "ZW",
|
||||
"es-AR": "AR",
|
||||
"es-BO": "BO",
|
||||
"es-CL": "CL",
|
||||
"es-CO": "CO",
|
||||
"es-CR": "CR",
|
||||
"es-CU": "CU",
|
||||
"es-DO": "DO",
|
||||
"es-EC": "EC",
|
||||
"es-ES": "ES",
|
||||
"es-GT": "GT",
|
||||
"es-HN": "HN",
|
||||
"es-MX": "MX",
|
||||
"es-NI": "NI",
|
||||
"es-PA": "PA",
|
||||
"es-PE": "PE",
|
||||
"es-PR": "PR",
|
||||
"es-PY": "PY",
|
||||
"es-SV": "SV",
|
||||
"es-US": "US",
|
||||
"es-UY": "UY",
|
||||
"es-VE": "VE",
|
||||
"et-EE": "EE",
|
||||
"eu-ES": "ES",
|
||||
"fa-AF": "AF",
|
||||
"fi-FI": "FI",
|
||||
"fil-PH": "PH",
|
||||
"fr-BE": "BE",
|
||||
"fr-BF": "BF",
|
||||
"fr-BI": "BI",
|
||||
"fr-BJ": "BJ",
|
||||
"fr-CA": "CA",
|
||||
"fr-CD": "CD",
|
||||
"fr-CF": "CF",
|
||||
"fr-CG": "CG",
|
||||
"fr-CH": "CH",
|
||||
"fr-CI": "CI",
|
||||
"fr-CM": "CM",
|
||||
"fr-DJ": "DJ",
|
||||
"fr-DZ": "DZ",
|
||||
"fr-FR": "FR",
|
||||
"fr-GA": "GA",
|
||||
"fr-HT": "HT",
|
||||
"fr-LU": "LU",
|
||||
"fr-MA": "MA",
|
||||
"fr-MG": "MG",
|
||||
"fr-ML": "ML",
|
||||
"fr-MU": "MU",
|
||||
"fr-NE": "NE",
|
||||
"fr-RW": "RW",
|
||||
"fr-SC": "SC",
|
||||
"fr-SN": "SN",
|
||||
"fr-TD": "TD",
|
||||
"fr-TG": "TG",
|
||||
"fr-TN": "TN",
|
||||
"fr-VU": "VU",
|
||||
"fy-NL": "NL",
|
||||
"ga-GB": "GB",
|
||||
"ga-IE": "IE",
|
||||
"gaa-GH": "GH",
|
||||
"gd-GB": "GB",
|
||||
"gl-ES": "ES",
|
||||
"gn-PY": "PY",
|
||||
"gu-IN": "IN",
|
||||
"haw-US": "US",
|
||||
"he-IL": "IL",
|
||||
"hi-IN": "IN",
|
||||
"hr-BA": "BA",
|
||||
"hr-HR": "HR",
|
||||
"ht-HT": "HT",
|
||||
"hu-HU": "HU",
|
||||
"hy-AM": "AM",
|
||||
"id-ID": "ID",
|
||||
"is-IS": "IS",
|
||||
"it-CH": "CH",
|
||||
"it-IT": "IT",
|
||||
"it-SM": "SM",
|
||||
"ja-JP": "JP",
|
||||
"kk-KZ": "KZ",
|
||||
"kn-IN": "IN",
|
||||
"ko-KR": "KR",
|
||||
"ky-KG": "KG",
|
||||
"ln-CD": "CD",
|
||||
"lt-LT": "LT",
|
||||
"lv-LV": "LV",
|
||||
"mg-MG": "MG",
|
||||
"mi-NZ": "NZ",
|
||||
"ml-IN": "IN",
|
||||
"mr-IN": "IN",
|
||||
"ms-SG": "SG",
|
||||
"mt-MT": "MT",
|
||||
"nb-NO": "NO",
|
||||
"ne-IN": "IN",
|
||||
"nl-BE": "BE",
|
||||
"nl-NL": "NL",
|
||||
"nl-SR": "SR",
|
||||
"nn-NO": "NO",
|
||||
"nso-ZA": "ZA",
|
||||
"ny-MW": "MW",
|
||||
"oc-ES": "ES",
|
||||
"or-IN": "IN",
|
||||
"pa-IN": "IN",
|
||||
"pl-PL": "PL",
|
||||
"ps-AF": "AF",
|
||||
"pt-AO": "AO",
|
||||
"pt-BR": "BR",
|
||||
"pt-CV": "CV",
|
||||
"pt-MZ": "MZ",
|
||||
"pt-PT": "PT",
|
||||
"pt-ST": "ST",
|
||||
"pt-TL": "TL",
|
||||
"qu-BO": "BO",
|
||||
"qu-EC": "EC",
|
||||
"qu-PE": "PE",
|
||||
"rm-CH": "CH",
|
||||
"rn-BI": "BI",
|
||||
"ro-MD": "MD",
|
||||
"ro-RO": "RO",
|
||||
"ru-BY": "BY",
|
||||
"ru-KG": "KG",
|
||||
"ru-KZ": "KZ",
|
||||
"ru-RU": "RU",
|
||||
"ru-UA": "UA",
|
||||
"rw-RW": "RW",
|
||||
"sd-IN": "IN",
|
||||
"sk-SK": "SK",
|
||||
"sl-SI": "SI",
|
||||
"sn-ZW": "ZW",
|
||||
"so-SO": "SO",
|
||||
"sr-BA": "BA",
|
||||
"sr-RS": "RS",
|
||||
"st-LS": "LS",
|
||||
"st-ZA": "ZA",
|
||||
"sv-FI": "FI",
|
||||
"sv-SE": "SE",
|
||||
"sw-CD": "CD",
|
||||
"sw-KE": "KE",
|
||||
"sw-TZ": "TZ",
|
||||
"sw-UG": "UG",
|
||||
"ta-IN": "IN",
|
||||
"ta-SG": "SG",
|
||||
"te-IN": "IN",
|
||||
"th-TH": "TH",
|
||||
"tn-BW": "BW",
|
||||
"tn-ZA": "ZA",
|
||||
"to-TO": "TO",
|
||||
"tr-CY": "CY",
|
||||
"tr-TR": "TR",
|
||||
"tt-RU": "RU",
|
||||
"uk-UA": "UA",
|
||||
"ur-IN": "IN",
|
||||
"ur-PK": "PK",
|
||||
"vi-VN": "VN",
|
||||
"wo-SN": "SN",
|
||||
"xh-ZA": "ZA",
|
||||
"yo-NG": "NG",
|
||||
"zh-CN": "HK",
|
||||
"zh-HK": "HK",
|
||||
"zh-SG": "SG",
|
||||
"zh-TW": "TW",
|
||||
"zu-ZA": "ZA"
|
||||
}
|
||||
},
|
||||
"google images": {
|
||||
"all_locale": "ZZ",
|
||||
"custom": {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -32,7 +32,6 @@ from searx.utils import (
|
||||
eval_xpath_getindex,
|
||||
eval_xpath_list,
|
||||
extract_text,
|
||||
gen_gsa_useragent,
|
||||
)
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
@@ -197,7 +196,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
# https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages
|
||||
|
||||
# https://github.com/searxng/searxng/issues/2515#issuecomment-1607150817
|
||||
ret_val["params"]["hl"] = f"{lang_code}-{country}"
|
||||
ret_val["params"]["hl"] = f"{lang_code}"
|
||||
|
||||
# lr parameter:
|
||||
# The lr (language restrict) parameter restricts search results to
|
||||
@@ -268,7 +267,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
|
||||
# HTTP headers
|
||||
|
||||
ret_val["headers"]["Accept"] = "*/*"
|
||||
ret_val["headers"]["User-Agent"] = gen_gsa_useragent()
|
||||
|
||||
# Cookies
|
||||
|
||||
|
||||
158
searx/engines/google_cse.py
Normal file
158
searx/engines/google_cse.py
Normal file
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Google Custom Search Engine"""
|
||||
|
||||
import datetime
|
||||
import typing as t
|
||||
from json import loads
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.exceptions import SearxEngineAPIException, SearxEngineTooManyRequestsException
|
||||
from searx.network import get
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
from searx.engines.google import fetch_traits # pylint: disable=unused-import
|
||||
from searx.engines.google import filter_mapping, get_google_info
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
about = {
|
||||
"website": "https://www.google.com",
|
||||
"wikidata_id": "Q2233943",
|
||||
"official_api_documentation": "https://developers.google.com/custom-search/docs/element",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "JSONP",
|
||||
"description": "Platform for creating custom search engines based on Google Search.",
|
||||
}
|
||||
|
||||
categories = ["general", "web"]
|
||||
paging = True
|
||||
max_page = 5
|
||||
page_size = 20
|
||||
time_range_support = True
|
||||
language_support = True
|
||||
safesearch = True
|
||||
|
||||
CX = "partner-pub-8993703457585266:4862972284" # blackle.com
|
||||
|
||||
CACHE: EngineCache
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
|
||||
|
||||
def _cse_token() -> dict[str, str]:
|
||||
token: dict[str, str] = CACHE.get(CX)
|
||||
if token:
|
||||
return token
|
||||
|
||||
resp = get(f"https://www.google.com/cse/cse.js?cx={CX}", timeout=10)
|
||||
if not resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain cse token")
|
||||
|
||||
end = resp.text.rfind("});")
|
||||
start = resp.text.rfind("({")
|
||||
opts: dict[str, str] = loads(resp.text[start + 1 : end + 1])
|
||||
|
||||
cse_tok = opts.get("cse_token")
|
||||
if not cse_tok:
|
||||
raise SearxEngineAPIException("failed to obtain cse token")
|
||||
|
||||
exp = opts.get("exp")
|
||||
token = {
|
||||
"cse_tok": cse_tok,
|
||||
"cselibv": opts.get("cselibVersion", ""),
|
||||
"exp": ",".join(exp) if exp else "",
|
||||
}
|
||||
CACHE.set(CX, token, expire=3600)
|
||||
return token
|
||||
|
||||
|
||||
def _get_start_and_end_date_str(time_range: str) -> tuple[str, str]:
|
||||
time_range_map = {"day": 1, "week": 7, "month": 30, "year": 365}
|
||||
|
||||
end_date = datetime.datetime.now()
|
||||
start_date = end_date - datetime.timedelta(days=time_range_map[time_range])
|
||||
|
||||
return start_date.strftime("%Y%m%d"), end_date.strftime("%Y%m%d")
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
token = _cse_token()
|
||||
|
||||
google_info = get_google_info(params, traits)
|
||||
info: dict[str, str] = google_info["params"]
|
||||
|
||||
hl = info["hl"]
|
||||
if params.get("searxng_locale", "all") == "all" or "ZZ" in hl:
|
||||
hl = "en"
|
||||
|
||||
args = {
|
||||
"rsz": "filtered_cse",
|
||||
"num": str(page_size),
|
||||
"hl": hl,
|
||||
"cselibv": token["cselibv"],
|
||||
"cx": CX,
|
||||
"q": query,
|
||||
"safe": filter_mapping[params["safesearch"]],
|
||||
"cse_tok": token["cse_tok"],
|
||||
"callback": "_",
|
||||
"rurl": "",
|
||||
}
|
||||
if params["time_range"]:
|
||||
start_date, end_date = _get_start_and_end_date_str(params["time_range"])
|
||||
args["sort"] = f"date:r:{start_date}:{end_date}"
|
||||
|
||||
if info.get("lr"):
|
||||
args["lr"] = info["lr"]
|
||||
if info.get("cr"):
|
||||
args["cr"] = info["cr"]
|
||||
if google_info["country"] not in (None, "ZZ"):
|
||||
args["gl"] = google_info["country"]
|
||||
if token["exp"]:
|
||||
args["exp"] = token["exp"]
|
||||
|
||||
start = (params["pageno"] - 1) * page_size
|
||||
if start:
|
||||
args["start"] = str(start)
|
||||
|
||||
params["url"] = "https://cse.google.com/cse/element/v1?" + urlencode(args)
|
||||
params["cookies"] = google_info["cookies"]
|
||||
params["headers"].update(google_info["headers"])
|
||||
params["headers"]["Referer"] = "https://cse.google.com/"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
json_resp = resp.text[resp.text.find("{") : resp.text.rfind("}") + 1]
|
||||
data = loads(json_resp)
|
||||
|
||||
# not the real types, but a sufficient approximation
|
||||
item: dict[str, str]
|
||||
error: dict[str, str | int]
|
||||
|
||||
if error := data.get("error"):
|
||||
message = error.get("message", "unknown error")
|
||||
if error.get("code") == 429:
|
||||
raise SearxEngineTooManyRequestsException(message=f"google cse: {message}")
|
||||
raise SearxEngineAPIException(f"google cse: {message}")
|
||||
|
||||
results = EngineResults()
|
||||
for item in data.get("results", []):
|
||||
url = item.get("unescapedUrl")
|
||||
if not url:
|
||||
continue
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
url=url,
|
||||
title=item.get("titleNoFormatting", ""),
|
||||
content=item.get("contentNoFormatting", ""),
|
||||
thumbnail=item.get("richSnippet", {}).get("cseThumbnail", {}).get("src", ""), # type: ignore
|
||||
)
|
||||
)
|
||||
return results
|
||||
66
searx/engines/neocities.py
Normal file
66
searx/engines/neocities.py
Normal file
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Neocities_ is open source software for creating blogs.
|
||||
|
||||
.. _Neocities : https://github.com/neocities/neocities
|
||||
"""
|
||||
|
||||
from urllib.parse import urlencode
|
||||
import typing as t
|
||||
|
||||
from lxml import html
|
||||
|
||||
from searx.utils import eval_xpath, eval_xpath_list, extract_text
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from extended_types import SXNG_Response
|
||||
from search.processors import OnlineParams
|
||||
|
||||
|
||||
# Engine metadata
|
||||
about = {
|
||||
"website": "https://neocities.org/",
|
||||
"wikidata_id": "Q17071099",
|
||||
"official_api_documentation": None,
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": "HTML",
|
||||
}
|
||||
|
||||
# Engine configuration
|
||||
categories = ["general", "blogs"]
|
||||
paging = True
|
||||
|
||||
# Search URL
|
||||
base_url = "https://neocities.org"
|
||||
|
||||
results_xpath = "//div[@class='result-item']"
|
||||
url_xpath = './/div[@class="result-url"]/a/@href'
|
||||
title_xpath = './/h3[@class="result-title"]/a/text()'
|
||||
content_xpath = './/p[@class="result-snippet"]//text()'
|
||||
screenshot_xpath = './/a[@class="result-screenshot"]/img/@src'
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
query_params: dict[str, t.Any] = {"q": query}
|
||||
if params['pageno'] > 1:
|
||||
offset = (params["pageno"] - 1) * 100
|
||||
query_params["start"] = offset
|
||||
params["url"] = f"{base_url}/search?{urlencode(query_params)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
results = EngineResults()
|
||||
dom = html.fromstring(resp.text)
|
||||
|
||||
for result in eval_xpath_list(dom, results_xpath):
|
||||
results.add(
|
||||
results.types.MainResult(
|
||||
url=extract_text(eval_xpath(result, url_xpath)),
|
||||
title=extract_text(eval_xpath(result, title_xpath)) or "",
|
||||
content=extract_text(eval_xpath(result, content_xpath)) or "",
|
||||
thumbnail=base_url + (extract_text(eval_xpath(result, screenshot_xpath)) or ""),
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
@@ -1199,6 +1199,11 @@ engines:
|
||||
- name: google
|
||||
engine: google
|
||||
shortcut: go
|
||||
inactive: true
|
||||
|
||||
- name: google cse
|
||||
engine: google_cse
|
||||
shortcut: goc
|
||||
|
||||
- name: google images
|
||||
engine: google_images
|
||||
@@ -1211,6 +1216,7 @@ engines:
|
||||
- name: google videos
|
||||
engine: google_videos
|
||||
shortcut: gov
|
||||
inactive: true
|
||||
|
||||
- name: google scholar
|
||||
engine: google_scholar
|
||||
@@ -1691,6 +1697,12 @@ engines:
|
||||
shortcut: mwm
|
||||
disabled: true
|
||||
|
||||
- name: neocities
|
||||
engine: neocities
|
||||
shortcut: nc
|
||||
disabled: true
|
||||
inactive: true
|
||||
|
||||
- name: niconico
|
||||
engine: niconico
|
||||
shortcut: nico
|
||||
|
||||
@@ -26,7 +26,7 @@ from lxml.etree import XPath, XPathError, XPathSyntaxError
|
||||
from lxml.etree import ElementBase, _Element # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
from searx import settings
|
||||
from searx.data import USER_AGENTS, gsa_useragents_loader
|
||||
from searx.data import USER_AGENTS
|
||||
from searx.version import VERSION_TAG
|
||||
from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
|
||||
from searx import logger
|
||||
@@ -82,14 +82,6 @@ def gen_useragent(os_string: str | None = None) -> str:
|
||||
)
|
||||
|
||||
|
||||
def gen_gsa_useragent() -> str:
|
||||
"""Return a random "Google Go App" User Agent suitable for Google
|
||||
|
||||
See searx/data/gsa_useragents.txt
|
||||
"""
|
||||
return choice(gsa_useragents_loader()) + " NSTNWV"
|
||||
|
||||
|
||||
class HTMLTextExtractor(HTMLParser):
|
||||
"""Internal class to extract text from HTML"""
|
||||
|
||||
|
||||
@@ -131,8 +131,8 @@ def fetch_traits_map() -> EngineTraitsMap:
|
||||
def filter_locales(traits_map: EngineTraitsMap) -> set[str]:
|
||||
"""Filter language & region tags by a threshold."""
|
||||
|
||||
min_eng_per_region = 18
|
||||
min_eng_per_lang = 23
|
||||
min_eng_per_region = 19
|
||||
min_eng_per_lang = 24
|
||||
|
||||
_: dict[str, int] = {}
|
||||
for eng in traits_map.values():
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""This script fetches user agents suitable for Google.
|
||||
|
||||
Output file: :origin:`searx/data/gsa_useragents.txt` (:origin:`CI Update data
|
||||
... <.github/workflows/data-update.yml>`).
|
||||
|
||||
.. Source for user agents: https://github.com/intoli/user-agents/
|
||||
|
||||
"""
|
||||
|
||||
from gzip import decompress
|
||||
from json import loads
|
||||
|
||||
from searx.data import data_dir
|
||||
from searx.network import get as http_get
|
||||
from searx.utils import searxng_useragent
|
||||
|
||||
DATA_FILE = data_dir / "gsa_useragents.txt"
|
||||
URL = "https://raw.githubusercontent.com/intoli/user-agents/main/src/user-agents.json.gz"
|
||||
|
||||
|
||||
def fetch_gsa_useragents() -> list[str]:
|
||||
response = http_get(URL, timeout=3.0, headers={"User-Agent": searxng_useragent()})
|
||||
response.raise_for_status()
|
||||
|
||||
suas: set[str] = set()
|
||||
for ua in loads(decompress(response.content)):
|
||||
if (
|
||||
"Android" in ua["userAgent"]
|
||||
and "Chrome" in ua["userAgent"]
|
||||
and "Samsung" not in ua["userAgent"]
|
||||
and "Android 10; K" not in ua["userAgent"]
|
||||
):
|
||||
suas.add(ua["userAgent"])
|
||||
|
||||
luas = list(suas)
|
||||
luas.sort()
|
||||
|
||||
return luas
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
useragents = fetch_gsa_useragents()
|
||||
with DATA_FILE.open("w", encoding="utf-8") as f:
|
||||
f.write("\n".join(useragents))
|
||||
@@ -7,7 +7,6 @@ data.:
|
||||
all : update searx/sxng_locales.py and searx/data/*
|
||||
traits : update searx/data/engine_traits.json & searx/sxng_locales.py
|
||||
useragents : update searx/data/useragents.json with the most recent versions of Firefox
|
||||
gsa_useragents: update searx/data/gsa_useragents.txt with compatible useragents
|
||||
locales : update searx/data/locales.json from babel
|
||||
currencies : update searx/data/currencies.json from wikidata
|
||||
EOF
|
||||
@@ -20,7 +19,6 @@ data.all() {
|
||||
pyenv.activate
|
||||
data.traits
|
||||
data.useragents
|
||||
data.gsa_useragents
|
||||
data.locales
|
||||
|
||||
build_msg DATA "update searx/data/osm_keys_tags.json"
|
||||
@@ -55,12 +53,6 @@ data.useragents() {
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
data.gsa_useragents() {
|
||||
build_msg DATA "update searx/data/gsa_useragents.txt"
|
||||
pyenv.cmd python searxng_extra/update/update_gsa_useragents.py
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
data.locales() {
|
||||
(
|
||||
set -e
|
||||
|
||||
Reference in New Issue
Block a user