jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2025-12-29 23:20:02 +00:00
Code Issues Projects Releases Wiki Activity

[mod] DuckDuckGo: reversed engineered & upgrade to data_type: traits_v1

Browse Source 
Partial reverse engineering of the DuckDuckGo (DDG) engines including a
improved language and region handling based on the enigne.traits_v1 data.

- DDG Lite
- DDG Instant Answer API
- DDG Images
- DDG Weather

docs/src/searx.engine.duckduckgo.rst:
  Online documentation of the DDG engines (make docs.live)

searx/data/engine_traits.json
  Add data type "traits_v1" generated by the fetch_traits() functions from:

  - "duckduckgo" (WEB),
  - "duckduckgo images" and
  - "duckduckgo weather"

  and remove data from obsolete data type "supported_languages".

searx/autocomplete.py:
  Reversed engineered Autocomplete from DDG.  Supports DDG's languages.

searx/engines/duckduckgo.py:
  - fetch_traits():  Fetch languages & regions from DDG.

  - get_ddg_lang(): Get DDG's language identifier from SearXNG's locale.  DDG
    defines its languages by region codes.  DDG-Lite does not offer a language
    selection to the user, only a region can be selected by the user.

  - Cache ``vqd`` value: The vqd value depends on the query string and is needed
    for the follow up pages or the images loaded by a XMLHttpRequest (DDG
    images).  The ``vqd`` value of a search term is stored for 10min in the
    redis DB.

  - DDG Lite engine: reversed engineered request method with improved Language
    and region support and better ``vqd`` handling.

searx/engines/duckduckgo_definitions.py: DDG Instant Answer API
  The *instant answers* API does not support languages, or at least we could not
  find out how language support should work.  It seems that most of the features
  are based on English terms.

searx/engines/duckduckgo_images.py: DDG Images
  Reversed engineered request method.  Improved language and region handling
  based on cookies and the enigne.traits_v1 data.  Response: add image format to
  the result list

searx/engines/duckduckgo_weather.py: DDG Weather
  Improved language and region handling based on cookies and the
  enigne.traits_v1 data.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
2022-11-05 15:10:52 +01:00
parent e9afc4f8ce
commit c80e82a855
8 changed files with 775 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files

633
searx/data/engine_traits.json
View File

@@ -471,145 +471,106 @@
"zh_TW"
]
},
"ddg definitions": {
"all_locale": "wt-wt",
"custom": {},
"data_type": "supported_languages",
"languages": {},
"regions": {
"ar-SA": "xa-ar",
"bg-BG": "bg-bg",
"ca-ES": "es-ca",
"cs-CZ": "cz-cs",
"da-DK": "dk-da",
"de-AT": "at-de",
"de-CH": "ch-de",
"de-DE": "de-de",
"el-GR": "gr-el",
"en-AU": "au-en",
"en-CA": "ca-en",
"en-GB": "uk-en",
"en-IE": "ie-en",
"en-IL": "il-en",
"en-IN": "in-en",
"en-MY": "my-en",
"en-NZ": "nz-en",
"en-PH": "ph-en",
"en-PK": "pk-en",
"en-SG": "sg-en",
"en-US": "us-en",
"en-ZA": "za-en",
"es-AR": "ar-es",
"es-CL": "cl-es",
"es-CO": "co-es",
"es-ES": "es-es",
"es-MX": "mx-es",
"es-PE": "pe-es",
"es-US": "us-es",
"et-EE": "ee-et",
"fi-FI": "fi-fi",
"fr-BE": "be-fr",
"fr-CA": "ca-fr",
"fr-CH": "ch-fr",
"fr-FR": "fr-fr",
"hr-HR": "hr-hr",
"hu-HU": "hu-hu",
"id-ID": "id-en",
"it-IT": "it-it",
"ja-JP": "jp-jp",
"ko-KR": "kr-kr",
"lt-LT": "lt-lt",
"lv-LV": "lv-lv",
"nb-NO": "no-no",
"nl-BE": "be-nl",
"nl-NL": "nl-nl",
"pl-PL": "pl-pl",
"pt-BR": "br-pt",
"pt-PT": "pt-pt",
"ro-RO": "ro-ro",
"ru-RU": "ru-ru",
"sk-SK": "sk-sk",
"sl-SI": "sl-sl",
"sv-SE": "se-sv",
"th-TH": "th-en",
"tr-TR": "tr-tr",
"uk-UA": "ua-uk",
"vi-VN": "vn-en",
"zh-CN": "cn-zh",
"zh-HK": "hk-tzh",
"zh-TW": "tw-tzh"
},
"supported_languages": [
"ar-XA",
"bg-BG",
"ca-CT",
"ca-ES",
"cs-CZ",
"da-DK",
"de-AT",
"de-CH",
"de-DE",
"el-GR",
"en-AU",
"en-CA",
"en-ID",
"en-IE",
"en-IL",
"en-IN",
"en-MY",
"en-NZ",
"en-PH",
"en-PK",
"en-SG",
"en-TH",
"en-UK",
"en-US",
"en-VN",
"en-ZA",
"es-AR",
"es-CL",
"es-CO",
"es-ES",
"es-MX",
"es-PE",
"es-US",
"et-EE",
"fi-FI",
"fr-BE",
"fr-CA",
"fr-CH",
"fr-FR",
"hr-HR",
"hu-HU",
"it-IT",
"jp-JP",
"kr-KR",
"lt-LT",
"lv-LV",
"nl-BE",
"nl-NL",
"no-NO",
"pl-PL",
"pt-BR",
"pt-PT",
"ro-RO",
"ru-RU",
"sk-SK",
"sl-SL",
"sv-SE",
"tr-TR",
"tzh-HK",
"tzh-TW",
"uk-UA",
"wt-WT",
"zh-CN"
]
},
"duckduckgo": {
"all_locale": "wt-wt",
"custom": {},
"data_type": "supported_languages",
"languages": {},
"custom": {
"lang_region": {
"ar-DZ": "ar_DZ",
"ar-JO": "ar_JO",
"ar-SA": "ar_SA",
"bn-IN": "bn_IN",
"de-CH": "de_CH",
"en-AU": "en_AU",
"en-CA": "en_CA",
"en-GB": "en_GB",
"es-AR": "es_AR",
"es-CL": "es_CL",
"es-CO": "es_CO",
"es-CR": "es_CR",
"es-EC": "es_EC",
"es-MX": "es_MX",
"es-PE": "es_PE",
"es-UY": "es_UY",
"es-VE": "es_VE",
"fr-BE": "fr_BE",
"fr-CA": "fr_CA",
"fr-CH": "fr_CH",
"nl-BE": "nl_BE",
"pt-BR": "pt_BR"
}
},
"data_type": "traits_v1",
"languages": {
"af": "af_ZA",
"ar": "ar_EG",
"ast": "ast_ES",
"az_Latn": "az_AZ",
"be": "be_BY",
"bg": "bg_BG",
"bn": "bn_BD",
"br": "br_FR",
"bs_Latn": "bs_BA",
"ca": "ca_ES",
"cs": "cs_CZ",
"cy": "cy_GB",
"da": "da_DK",
"de": "de_DE",
"el": "el_GR",
"en": "en_US",
"eo": "eo_XX",
"es": "es_ES",
"et": "et_EE",
"eu": "eu_ES",
"fa": "fa_IR",
"fi": "fi_FI",
"fil": "tl_PH",
"fr": "fr_FR",
"ga": "ga_IE",
"gd": "gd_GB",
"gl": "gl_ES",
"he": "he_IL",
"hi": "hi_IN",
"hr": "hr_HR",
"hu": "hu_HU",
"hy": "hy_AM",
"id": "id_ID",
"is": "is_IS",
"it": "it_IT",
"ja": "ja_JP",
"kab": "kab_DZ",
"kn": "kn_IN",
"ko": "ko_KR",
"ku": "ku",
"kw": "kw_GB",
"lt": "lt_LT",
"lv": "lv_LV",
"ml": "ml_IN",
"mr": "mr_IN",
"ms": "ms_MY",
"nb": "nb_NO",
"nl": "nl_NL",
"nn": "nn_NO",
"pl": "pl_PL",
"pt": "pt_PT",
"ro": "ro_RO",
"ru": "ru_RU",
"sc": "sc_IT",
"si": "si_LK",
"sk": "sk_SK",
"sl": "sl_SI",
"sq": "sq_AL",
"sr_Cyrl": "sr_RS",
"sv": "sv_SE",
"ta": "ta_IN",
"te": "te_IN",
"th": "th_TH",
"tr": "tr_TR",
"uk": "uk_UA",
"ur": "ur_PK",
"vi": "vi_VN",
"zh_Hans": "zh_CN",
"zh_Hant": "zh_TW"
},
"regions": {
"ar-SA": "xa-ar",
"bg-BG": "bg-bg",
@@ -673,77 +634,108 @@
"zh-HK": "hk-tzh",
"zh-TW": "tw-tzh"
},
"supported_languages": [
"ar-XA",
"bg-BG",
"ca-CT",
"ca-ES",
"cs-CZ",
"da-DK",
"de-AT",
"de-CH",
"de-DE",
"el-GR",
"en-AU",
"en-CA",
"en-ID",
"en-IE",
"en-IL",
"en-IN",
"en-MY",
"en-NZ",
"en-PH",
"en-PK",
"en-SG",
"en-TH",
"en-UK",
"en-US",
"en-VN",
"en-ZA",
"es-AR",
"es-CL",
"es-CO",
"es-ES",
"es-MX",
"es-PE",
"es-US",
"et-EE",
"fi-FI",
"fr-BE",
"fr-CA",
"fr-CH",
"fr-FR",
"hr-HR",
"hu-HU",
"it-IT",
"jp-JP",
"kr-KR",
"lt-LT",
"lv-LV",
"nl-BE",
"nl-NL",
"no-NO",
"pl-PL",
"pt-BR",
"pt-PT",
"ro-RO",
"ru-RU",
"sk-SK",
"sl-SL",
"sv-SE",
"tr-TR",
"tzh-HK",
"tzh-TW",
"uk-UA",
"wt-WT",
"zh-CN"
]
"supported_languages": {}
},
"duckduckgo images": {
"all_locale": "wt-wt",
"custom": {},
"data_type": "supported_languages",
"languages": {},
"custom": {
"lang_region": {
"ar-DZ": "ar_DZ",
"ar-JO": "ar_JO",
"ar-SA": "ar_SA",
"bn-IN": "bn_IN",
"de-CH": "de_CH",
"en-AU": "en_AU",
"en-CA": "en_CA",
"en-GB": "en_GB",
"es-AR": "es_AR",
"es-CL": "es_CL",
"es-CO": "es_CO",
"es-CR": "es_CR",
"es-EC": "es_EC",
"es-MX": "es_MX",
"es-PE": "es_PE",
"es-UY": "es_UY",
"es-VE": "es_VE",
"fr-BE": "fr_BE",
"fr-CA": "fr_CA",
"fr-CH": "fr_CH",
"nl-BE": "nl_BE",
"pt-BR": "pt_BR"
}
},
"data_type": "traits_v1",
"languages": {
"af": "af_ZA",
"ar": "ar_EG",
"ast": "ast_ES",
"az_Latn": "az_AZ",
"be": "be_BY",
"bg": "bg_BG",
"bn": "bn_BD",
"br": "br_FR",
"bs_Latn": "bs_BA",
"ca": "ca_ES",
"cs": "cs_CZ",
"cy": "cy_GB",
"da": "da_DK",
"de": "de_DE",
"el": "el_GR",
"en": "en_US",
"eo": "eo_XX",
"es": "es_ES",
"et": "et_EE",
"eu": "eu_ES",
"fa": "fa_IR",
"fi": "fi_FI",
"fil": "tl_PH",
"fr": "fr_FR",
"ga": "ga_IE",
"gd": "gd_GB",
"gl": "gl_ES",
"he": "he_IL",
"hi": "hi_IN",
"hr": "hr_HR",
"hu": "hu_HU",
"hy": "hy_AM",
"id": "id_ID",
"is": "is_IS",
"it": "it_IT",
"ja": "ja_JP",
"kab": "kab_DZ",
"kn": "kn_IN",
"ko": "ko_KR",
"ku": "ku",
"kw": "kw_GB",
"lt": "lt_LT",
"lv": "lv_LV",
"ml": "ml_IN",
"mr": "mr_IN",
"ms": "ms_MY",
"nb": "nb_NO",
"nl": "nl_NL",
"nn": "nn_NO",
"pl": "pl_PL",
"pt": "pt_PT",
"ro": "ro_RO",
"ru": "ru_RU",
"sc": "sc_IT",
"si": "si_LK",
"sk": "sk_SK",
"sl": "sl_SI",
"sq": "sq_AL",
"sr_Cyrl": "sr_RS",
"sv": "sv_SE",
"ta": "ta_IN",
"te": "te_IN",
"th": "th_TH",
"tr": "tr_TR",
"uk": "uk_UA",
"ur": "ur_PK",
"vi": "vi_VN",
"zh_Hans": "zh_CN",
"zh_Hant": "zh_TW"
},
"regions": {
"ar-SA": "xa-ar",
"bg-BG": "bg-bg",
@@ -807,71 +799,172 @@
"zh-HK": "hk-tzh",
"zh-TW": "tw-tzh"
},
"supported_languages": [
"ar-XA",
"bg-BG",
"ca-CT",
"ca-ES",
"cs-CZ",
"da-DK",
"de-AT",
"de-CH",
"de-DE",
"el-GR",
"en-AU",
"en-CA",
"en-ID",
"en-IE",
"en-IL",
"en-IN",
"en-MY",
"en-NZ",
"en-PH",
"en-PK",
"en-SG",
"en-TH",
"en-UK",
"en-US",
"en-VN",
"en-ZA",
"es-AR",
"es-CL",
"es-CO",
"es-ES",
"es-MX",
"es-PE",
"es-US",
"et-EE",
"fi-FI",
"fr-BE",
"fr-CA",
"fr-CH",
"fr-FR",
"hr-HR",
"hu-HU",
"it-IT",
"jp-JP",
"kr-KR",
"lt-LT",
"lv-LV",
"nl-BE",
"nl-NL",
"no-NO",
"pl-PL",
"pt-BR",
"pt-PT",
"ro-RO",
"ru-RU",
"sk-SK",
"sl-SL",
"sv-SE",
"tr-TR",
"tzh-HK",
"tzh-TW",
"uk-UA",
"wt-WT",
"zh-CN"
]
"supported_languages": {}
},
"duckduckgo weather": {
"all_locale": "wt-wt",
"custom": {
"lang_region": {
"ar-DZ": "ar_DZ",
"ar-JO": "ar_JO",
"ar-SA": "ar_SA",
"bn-IN": "bn_IN",
"de-CH": "de_CH",
"en-AU": "en_AU",
"en-CA": "en_CA",
"en-GB": "en_GB",
"es-AR": "es_AR",
"es-CL": "es_CL",
"es-CO": "es_CO",
"es-CR": "es_CR",
"es-EC": "es_EC",
"es-MX": "es_MX",
"es-PE": "es_PE",
"es-UY": "es_UY",
"es-VE": "es_VE",
"fr-BE": "fr_BE",
"fr-CA": "fr_CA",
"fr-CH": "fr_CH",
"nl-BE": "nl_BE",
"pt-BR": "pt_BR"
}
},
"data_type": "traits_v1",
"languages": {
"af": "af_ZA",
"ar": "ar_EG",
"ast": "ast_ES",
"az_Latn": "az_AZ",
"be": "be_BY",
"bg": "bg_BG",
"bn": "bn_BD",
"br": "br_FR",
"bs_Latn": "bs_BA",
"ca": "ca_ES",
"cs": "cs_CZ",
"cy": "cy_GB",
"da": "da_DK",
"de": "de_DE",
"el": "el_GR",
"en": "en_US",
"eo": "eo_XX",
"es": "es_ES",
"et": "et_EE",
"eu": "eu_ES",
"fa": "fa_IR",
"fi": "fi_FI",
"fil": "tl_PH",
"fr": "fr_FR",
"ga": "ga_IE",
"gd": "gd_GB",
"gl": "gl_ES",
"he": "he_IL",
"hi": "hi_IN",
"hr": "hr_HR",
"hu": "hu_HU",
"hy": "hy_AM",
"id": "id_ID",
"is": "is_IS",
"it": "it_IT",
"ja": "ja_JP",
"kab": "kab_DZ",
"kn": "kn_IN",
"ko": "ko_KR",
"ku": "ku",
"kw": "kw_GB",
"lt": "lt_LT",
"lv": "lv_LV",
"ml": "ml_IN",
"mr": "mr_IN",
"ms": "ms_MY",
"nb": "nb_NO",
"nl": "nl_NL",
"nn": "nn_NO",
"pl": "pl_PL",
"pt": "pt_PT",
"ro": "ro_RO",
"ru": "ru_RU",
"sc": "sc_IT",
"si": "si_LK",
"sk": "sk_SK",
"sl": "sl_SI",
"sq": "sq_AL",
"sr_Cyrl": "sr_RS",
"sv": "sv_SE",
"ta": "ta_IN",
"te": "te_IN",
"th": "th_TH",
"tr": "tr_TR",
"uk": "uk_UA",
"ur": "ur_PK",
"vi": "vi_VN",
"zh_Hans": "zh_CN",
"zh_Hant": "zh_TW"
},
"regions": {
"ar-SA": "xa-ar",
"bg-BG": "bg-bg",
"ca-ES": "es-ca",
"cs-CZ": "cz-cs",
"da-DK": "dk-da",
"de-AT": "at-de",
"de-CH": "ch-de",
"de-DE": "de-de",
"el-GR": "gr-el",
"en-AU": "au-en",
"en-CA": "ca-en",
"en-GB": "uk-en",
"en-IE": "ie-en",
"en-IL": "il-en",
"en-IN": "in-en",
"en-MY": "my-en",
"en-NZ": "nz-en",
"en-PH": "ph-en",
"en-PK": "pk-en",
"en-SG": "sg-en",
"en-US": "us-en",
"en-ZA": "za-en",
"es-AR": "ar-es",
"es-CL": "cl-es",
"es-CO": "co-es",
"es-ES": "es-es",
"es-MX": "mx-es",
"es-PE": "pe-es",
"es-US": "us-es",
"et-EE": "ee-et",
"fi-FI": "fi-fi",
"fr-BE": "be-fr",
"fr-CA": "ca-fr",
"fr-CH": "ch-fr",
"fr-FR": "fr-fr",
"hr-HR": "hr-hr",
"hu-HU": "hu-hu",
"id-ID": "id-en",
"it-IT": "it-it",
"ja-JP": "jp-jp",
"ko-KR": "kr-kr",
"lt-LT": "lt-lt",
"lv-LV": "lv-lv",
"nb-NO": "no-no",
"nl-BE": "be-nl",
"nl-NL": "nl-nl",
"pl-PL": "pl-pl",
"pt-BR": "br-pt",
"pt-PT": "pt-pt",
"ro-RO": "ro-ro",
"ru-RU": "ru-ru",
"sk-SK": "sk-sk",
"sl-SI": "sl-sl",
"sv-SE": "se-sv",
"th-TH": "th-en",
"tr-TR": "tr-tr",
"uk-UA": "ua-uk",
"vi-VN": "vn-en",
"zh-CN": "cn-zh",
"zh-HK": "hk-tzh",
"zh-TW": "tw-tzh"
},
"supported_languages": {}
},
"google": {
"all_locale": "ZZ",