From ca49650407a04cc0cc043759d5cb2cd2c92efd20 Mon Sep 17 00:00:00 2001 From: vojkovic Date: Tue, 15 Sep 2026 02:19:23 +0000 Subject: [PATCH] [fix] engines: yahoo modernisation and ybv cookie --- searx/engines/yahoo.py | 282 ++++++++++++++++++++--------------------- 1 file changed, 135 insertions(+), 147 deletions(-) diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index ca455c603..dffc17675 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -1,53 +1,51 @@ # SPDX-License-Identifier: AGPL-3.0-or-later """Yahoo Search (Web) -Languages are supported by mapping the language to a domain. If domain is not -found in :py:obj:`lang2domain` URL ``.search.yahoo.com`` is used. +As a bot detection measure, Yahoo has put search behind a cookie +named `YBV`, which is cached for 24h before expiring. """ -from urllib.parse import ( - unquote, - urlencode, -) +import typing as t +from urllib.parse import unquote, urlencode, urljoin + from lxml import html -from searx.utils import ( - eval_xpath_getindex, - eval_xpath_list, - extract_text, - html_to_text, -) +from searx.enginelib import EngineCache +from searx.network import get # see https://github.com/searxng/searxng/issues/762 +from searx.result_types import EngineResults +from searx.utils import eval_xpath_getindex, eval_xpath_list, extract_text, html_to_text + +if t.TYPE_CHECKING: + from searx.extended_types import SXNG_Response + from searx.search.processors import OnlineParams -# about about = { - "website": 'https://search.yahoo.com/', + "website": "https://search.yahoo.com/", "wikidata_id": None, - "official_api_documentation": 'https://developer.yahoo.com/api/', + "official_api_documentation": "https://developer.yahoo.com/api/", "use_official_api": False, "require_api_key": False, - "results": 'HTML', + "results": "HTML", } -# engine dependent config -categories = ['general', 'web'] +categories = ["general", "web"] paging = True time_range_support = True +safesearch = True -time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm'} -safesearch_dict = {0: 'p', 1: 'i', 2: 'r'} +time_range_dict = {"day": "d", "week": "w", "month": "m"} +safesearch_dict = {0: "p", 1: "i", 2: "r"} region2domain = { "CO": "co.search.yahoo.com", # Colombia "TH": "th.search.yahoo.com", # Thailand "VE": "ve.search.yahoo.com", # Venezuela "CL": "cl.search.yahoo.com", # Chile - "HK": "hk.search.yahoo.com", # Hong Kong "PE": "pe.search.yahoo.com", # Peru "CA": "ca.search.yahoo.com", # Canada "DE": "de.search.yahoo.com", # Germany "FR": "fr.search.yahoo.com", # France - "TW": "tw.search.yahoo.com", # Taiwan "GB": "uk.search.yahoo.com", # United Kingdom "UK": "uk.search.yahoo.com", "BR": "br.search.yahoo.com", # Brazil @@ -57,27 +55,28 @@ region2domain = { "AR": "ar.search.yahoo.com", # Argentina "MX": "mx.search.yahoo.com", # Mexico "SG": "sg.search.yahoo.com", # Singapore + "AU": "au.search.yahoo.com", # Australia + "NZ": "nz.search.yahoo.com", # New Zealand + "IE": "ie.search.yahoo.com", # Ireland + "ZA": "za.search.yahoo.com", # South Africa + "US": "search.yahoo.com", + "IT": "it.search.yahoo.com", # Italy + "NL": "nl.search.yahoo.com", # Netherlands + "BE": "be.search.yahoo.com", # Belgium + "CH": "ch.search.yahoo.com", # Switzerland + "AT": "at.search.yahoo.com", # Austria + "SE": "se.search.yahoo.com", # Sweden + "NO": "no.search.yahoo.com", # Norway + "DK": "dk.search.yahoo.com", # Denmark + "FI": "fi.search.yahoo.com", # Finland + "GR": "gr.search.yahoo.com", # Greece + "TR": "tr.search.yahoo.com", # Turkey + "MY": "malaysia.search.yahoo.com", # Malaysia + "ID": "id.search.yahoo.com", # Indonesia + "VN": "vn.search.yahoo.com", # Vietnam + "PL": "pl.search.yahoo.com", # Poland + "RO": "ro.search.yahoo.com", # Romania } -"""Map regions to domain""" - -lang2domain = { - 'zh_chs': 'hk.search.yahoo.com', - 'zh_cht': 'tw.search.yahoo.com', - 'any': 'search.yahoo.com', - 'en': 'search.yahoo.com', - 'bg': 'search.yahoo.com', - 'cs': 'search.yahoo.com', - 'da': 'search.yahoo.com', - 'el': 'search.yahoo.com', - 'et': 'search.yahoo.com', - 'he': 'search.yahoo.com', - 'hr': 'search.yahoo.com', - 'ja': 'search.yahoo.com', - 'ko': 'search.yahoo.com', - 'sk': 'search.yahoo.com', - 'sl': 'search.yahoo.com', -} -"""Map language to domain""" yahoo_languages = { "all": "any", @@ -111,94 +110,64 @@ yahoo_languages = { "sv": "sv", # Swedish "th": "th", # Thai "tr": "tr", # Turkish - "zh": "zh_chs", # Chinese (Simplified) - "zh_Hans": "zh_chs", - 'zh-CN': "zh_chs", - "zh_Hant": "zh_cht", # Chinese (Traditional) - "zh-HK": "zh_cht", - 'zh-TW': "zh_cht", + "zh": "zh_chs", } -def build_sb_cookie(cookie_params): - """Build sB cookie parameter from provided parameters. +CACHE: EngineCache +"""YBV cookie""" - :param cookie_params: Dictionary of cookie parameters - :type cookie_params: dict - :returns: Formatted cookie string - :rtype: str - - Example: - >>> cookie_params = {'v': '1', 'vm': 'p', 'fl': '1', 'vl': 'lang_fr'} - >>> build_sb_cookie(cookie_params) - 'v=1&vm=p&fl=1&vl=lang_fr' - """ - - cookie_parts = [] - for key, value in cookie_params.items(): - cookie_parts.append(f"{key}={value}") - - return "&".join(cookie_parts) +_YBV_HOPS = 4 +"""Tracking pixel -> gif -> (optional) geo redirect i.e. au.search.yahoo.com -> 200""" -def request(query, params): - """Build Yahoo search request.""" - - lang, region = (params["language"].split("-") + [None])[:2] - lang = yahoo_languages.get(lang, "any") - - # Build URL parameters - # - p (str): Search query string - # - btf (str): Time filter, maps to values like 'd' (day), 'w' (week), 'm' (month) - # - iscqry (str): Empty string, necessary for results to appear properly on first page - # - b (int): Search offset for pagination - # - pz (str): Amount of results expected for the page - url_params = {'p': query} - - btf = time_range_dict.get(params['time_range']) - if btf: - url_params['btf'] = btf - - if params['pageno'] == 1: - url_params['iscqry'] = '' - elif params['pageno'] >= 2: - url_params['b'] = params['pageno'] * 7 + 1 # 8, 15, 21, etc. - url_params['pz'] = 7 - url_params['bct'] = 0 - url_params['xargs'] = 0 - - # Build sB cookie (for filters) - # - vm (str): SafeSearch filter, maps to values like 'p' (None), 'i' (Moderate), 'r' (Strict) - # - fl (bool): Indicates if a search language is used or not - # - vl (str): The search language to use (e.g. lang_fr) - sbcookie_params = { - 'v': 1, - 'vm': safesearch_dict[params['safesearch']], - 'fl': 1, - 'vl': f'lang_{lang}', - 'pn': 10, - 'rw': 'new', - 'userset': 1, - } - params['cookies']['sB'] = build_sb_cookie(sbcookie_params) - - # Search region/language - domain = region2domain.get(region) - if not domain: - domain = lang2domain.get(lang, f'{lang}.search.yahoo.com') - logger.debug(f'domain selected: {domain}') - logger.debug(f'cookies: {params["cookies"]}') - - params['url'] = f'https://{domain}/search?{urlencode(url_params)}' - params['domain'] = domain +def setup(engine_settings: dict[str, t.Any]): + global CACHE # pylint: disable=global-statement + CACHE = EngineCache(engine_settings["name"]) -def parse_url(url_string): +def request(query: str, params: "OnlineParams") -> None: + parts = params["searxng_locale"].split("-") + lang = yahoo_languages.get(parts[0], "any") + + url_params: dict[str, str | int] = {"p": query} + if params["time_range"] in time_range_dict: + url_params["btf"] = time_range_dict[params["time_range"]] + if params["pageno"] == 1: + url_params["iscqry"] = "" + else: + url_params["b"] = params["pageno"] * 7 + 1 + url_params["pz"] = 7 + url_params["bct"] = 0 + url_params["xargs"] = 0 + + params["cookies"]["sB"] = urlencode( + { + "v": 1, + "vm": safesearch_dict[params["safesearch"]], + "fl": 1, + "vl": f"lang_{lang}", + "pn": 10, + "rw": "new", + "userset": 1, + } + ) + + domain = "search.yahoo.com" + if len(parts) > 1 and parts[-1] in region2domain: + domain = region2domain[parts[-1]] + logger.debug("domain selected: %s", domain) + params["url"] = f"https://{domain}/search?{urlencode(url_params)}" + if ybv := CACHE.get("YBV"): + params["cookies"]["YBV"] = ybv + + +def parse_url(url_string: str) -> str: """remove yahoo-specific tracking-url""" - endings = ['/RS', '/RK'] + endings = ["/RS", "/RK"] endpositions = [] - start = url_string.find('http', url_string.find('/RU=') + 1) + start = url_string.find("http", url_string.find("/RU=") + 1) for ending in endings: endpos = url_string.rfind(ending) @@ -212,45 +181,64 @@ def parse_url(url_string): return unquote(url_string[start:end]) -def response(resp): - """parse response""" +def _yahoo_html(resp: "SXNG_Response") -> "SXNG_Response": + cookies = dict(resp.search_params["cookies"]) + params = resp.search_params - results = [] + for _ in range(_YBV_HOPS): + if ybv := resp.cookies.get("YBV"): + cookies["YBV"] = ybv + if ybv.startswith("v0.2"): + CACHE.set("YBV", ybv, expire=86400) + + if resp.status_code == 200: + return resp + + loc = resp.headers.get("location") + if resp.status_code not in (302, 307) or not loc: + return resp + + # request ourselves instead of following it + resp = get( + urljoin(resp.url, loc), + cookies=cookies, + headers=params["headers"], + allow_redirects=False, + ) + resp.search_params = params + + return resp + + +def response(resp: "SXNG_Response") -> EngineResults: + resp = _yahoo_html(resp) + results = EngineResults() + if resp.status_code != 200: + resp.raise_for_status() dom = html.fromstring(resp.text) - url_xpath = './/div[contains(@class,"compTitle")]/h3/a/@href' - title_xpath = './/h3//a/@aria-label' - - domain = resp.search_params['domain'] - if domain == "search.yahoo.com": - url_xpath = './/div[contains(@class,"compTitle")]/a/@href' - title_xpath = './/div[contains(@class,"compTitle")]/a/h3/span' - - # parse results for result in eval_xpath_list(dom, '//div[contains(@class,"algo-sr")]'): - url = eval_xpath_getindex(result, url_xpath, 0, default=None) + url = eval_xpath_getindex(result, './/div[contains(@class,"compTitle")]//a/@href', 0, default=None) if url is None: continue url = parse_url(url) - - title = eval_xpath_getindex(result, title_xpath, 0, default='') - title: str = extract_text(title) - content = eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0, default='') - content: str = extract_text(content, allow_none=True) - - # append result - results.append( - { - 'url': url, + title = extract_text(eval_xpath_getindex(result, ".//h3//a/@aria-label", 0, default=None), allow_none=True) + if not title: + title = extract_text(eval_xpath_getindex(result, ".//a/h3", 0, default=None), allow_none=True) + if not title: + continue + content = extract_text(eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0)) + results.add( + results.types.MainResult( + url=url, # title sometimes contains HTML tags / see # https://github.com/searxng/searxng/issues/3790 - 'title': " ".join(html_to_text(title).strip().split()), - 'content': " ".join(html_to_text(content).strip().split()), - } + title=html_to_text(title), + content=html_to_text(content), + ) ) for suggestion in eval_xpath_list(dom, '//div[contains(@class, "AlsoTry")]//table//a'): - # append suggestion - results.append({'suggestion': extract_text(suggestion)}) + results.add(results.types.LegacyResult(suggestion=extract_text(suggestion))) return results