[fix] engines: yahoo modernisation and ybv cookie

searx/engines/yahoo.py

@@ -1,53 +1,51 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 """Yahoo Search (Web)
 
-Languages are supported by mapping the language to a domain.  If domain is not
+As a bot detection measure, Yahoo has put search behind a cookie
-found in :py:obj:`lang2domain` URL ``<lang>.search.yahoo.com`` is used.
+named `YBV`, which is cached for 24h before expiring.
 
 """
 
-from urllib.parse import (
+import typing as t
-    unquote,
+from urllib.parse import unquote, urlencode, urljoin
-    urlencode,
+
-)
 from lxml import html
 
-from searx.utils import (
+from searx.enginelib import EngineCache
-    eval_xpath_getindex,
+from searx.network import get  # see https://github.com/searxng/searxng/issues/762
-    eval_xpath_list,
+from searx.result_types import EngineResults
-    extract_text,
+from searx.utils import eval_xpath_getindex, eval_xpath_list, extract_text, html_to_text
-    html_to_text,
+
-)
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
 
-# about
 about = {
-    "website": 'https://search.yahoo.com/',
+    "website": "https://search.yahoo.com/",
     "wikidata_id": None,
-    "official_api_documentation": 'https://developer.yahoo.com/api/',
+    "official_api_documentation": "https://developer.yahoo.com/api/",
     "use_official_api": False,
     "require_api_key": False,
-    "results": 'HTML',
+    "results": "HTML",
 }
 
-# engine dependent config
+categories = ["general", "web"]
-categories = ['general', 'web']
 paging = True
 time_range_support = True
+safesearch = True
 
-time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm'}
+time_range_dict = {"day": "d", "week": "w", "month": "m"}
-safesearch_dict = {0: 'p', 1: 'i', 2: 'r'}
+safesearch_dict = {0: "p", 1: "i", 2: "r"}
 
 region2domain = {
     "CO": "co.search.yahoo.com",  # Colombia
     "TH": "th.search.yahoo.com",  # Thailand
     "VE": "ve.search.yahoo.com",  # Venezuela
     "CL": "cl.search.yahoo.com",  # Chile
-    "HK": "hk.search.yahoo.com",  # Hong Kong
     "PE": "pe.search.yahoo.com",  # Peru
     "CA": "ca.search.yahoo.com",  # Canada
     "DE": "de.search.yahoo.com",  # Germany
     "FR": "fr.search.yahoo.com",  # France
-    "TW": "tw.search.yahoo.com",  # Taiwan
     "GB": "uk.search.yahoo.com",  # United Kingdom
     "UK": "uk.search.yahoo.com",
     "BR": "br.search.yahoo.com",  # Brazil
@@ -57,27 +55,28 @@ region2domain = {
     "AR": "ar.search.yahoo.com",  # Argentina
     "MX": "mx.search.yahoo.com",  # Mexico
     "SG": "sg.search.yahoo.com",  # Singapore
+    "AU": "au.search.yahoo.com",  # Australia
+    "NZ": "nz.search.yahoo.com",  # New Zealand
+    "IE": "ie.search.yahoo.com",  # Ireland
+    "ZA": "za.search.yahoo.com",  # South Africa
+    "US": "search.yahoo.com",
+    "IT": "it.search.yahoo.com",  # Italy
+    "NL": "nl.search.yahoo.com",  # Netherlands
+    "BE": "be.search.yahoo.com",  # Belgium
+    "CH": "ch.search.yahoo.com",  # Switzerland
+    "AT": "at.search.yahoo.com",  # Austria
+    "SE": "se.search.yahoo.com",  # Sweden
+    "NO": "no.search.yahoo.com",  # Norway
+    "DK": "dk.search.yahoo.com",  # Denmark
+    "FI": "fi.search.yahoo.com",  # Finland
+    "GR": "gr.search.yahoo.com",  # Greece
+    "TR": "tr.search.yahoo.com",  # Turkey
+    "MY": "malaysia.search.yahoo.com",  # Malaysia
+    "ID": "id.search.yahoo.com",  # Indonesia
+    "VN": "vn.search.yahoo.com",  # Vietnam
+    "PL": "pl.search.yahoo.com",  # Poland
+    "RO": "ro.search.yahoo.com",  # Romania
 }
-"""Map regions to domain"""
-
-lang2domain = {
-    'zh_chs': 'hk.search.yahoo.com',
-    'zh_cht': 'tw.search.yahoo.com',
-    'any': 'search.yahoo.com',
-    'en': 'search.yahoo.com',
-    'bg': 'search.yahoo.com',
-    'cs': 'search.yahoo.com',
-    'da': 'search.yahoo.com',
-    'el': 'search.yahoo.com',
-    'et': 'search.yahoo.com',
-    'he': 'search.yahoo.com',
-    'hr': 'search.yahoo.com',
-    'ja': 'search.yahoo.com',
-    'ko': 'search.yahoo.com',
-    'sk': 'search.yahoo.com',
-    'sl': 'search.yahoo.com',
-}
-"""Map language to domain"""
 
 yahoo_languages = {
     "all": "any",
@@ -111,94 +110,64 @@ yahoo_languages = {
     "sv": "sv",  # Swedish
     "th": "th",  # Thai
     "tr": "tr",  # Turkish
-    "zh": "zh_chs",  # Chinese (Simplified)
+    "zh": "zh_chs",
-    "zh_Hans": "zh_chs",
-    'zh-CN': "zh_chs",
-    "zh_Hant": "zh_cht",  # Chinese (Traditional)
-    "zh-HK": "zh_cht",
-    'zh-TW': "zh_cht",
 }
 
 
-def build_sb_cookie(cookie_params):
+CACHE: EngineCache
-    """Build sB cookie parameter from provided parameters.
+"""YBV cookie"""
 
-    :param cookie_params: Dictionary of cookie parameters
+_YBV_HOPS = 4
-    :type cookie_params: dict
+"""Tracking pixel -> gif -> (optional) geo redirect i.e. au.search.yahoo.com -> 200"""
-    :returns: Formatted cookie string
-    :rtype: str
-
-    Example:
-        >>> cookie_params = {'v': '1', 'vm': 'p', 'fl': '1', 'vl': 'lang_fr'}
-        >>> build_sb_cookie(cookie_params)
-        'v=1&vm=p&fl=1&vl=lang_fr'
-    """
-
-    cookie_parts = []
-    for key, value in cookie_params.items():
-        cookie_parts.append(f"{key}={value}")
-
-    return "&".join(cookie_parts)
 
 
-def request(query, params):
+def setup(engine_settings: dict[str, t.Any]):
-    """Build Yahoo search request."""
+    global CACHE  # pylint: disable=global-statement
-
+    CACHE = EngineCache(engine_settings["name"])
-    lang, region = (params["language"].split("-") + [None])[:2]
-    lang = yahoo_languages.get(lang, "any")
-
-    # Build URL parameters
-    # - p (str): Search query string
-    # - btf (str): Time filter, maps to values like 'd' (day), 'w' (week), 'm' (month)
-    # - iscqry (str): Empty string, necessary for results to appear properly on first page
-    # - b (int): Search offset for pagination
-    # - pz (str): Amount of results expected for the page
-    url_params = {'p': query}
-
-    btf = time_range_dict.get(params['time_range'])
-    if btf:
-        url_params['btf'] = btf
-
-    if params['pageno'] == 1:
-        url_params['iscqry'] = ''
-    elif params['pageno'] >= 2:
-        url_params['b'] = params['pageno'] * 7 + 1  #  8, 15, 21, etc.
-        url_params['pz'] = 7
-        url_params['bct'] = 0
-        url_params['xargs'] = 0
-
-    # Build sB cookie (for filters)
-    # - vm (str): SafeSearch filter, maps to values like 'p' (None), 'i' (Moderate), 'r' (Strict)
-    # - fl (bool): Indicates if a search language is used or not
-    # - vl (str): The search language to use (e.g. lang_fr)
-    sbcookie_params = {
-        'v': 1,
-        'vm': safesearch_dict[params['safesearch']],
-        'fl': 1,
-        'vl': f'lang_{lang}',
-        'pn': 10,
-        'rw': 'new',
-        'userset': 1,
-    }
-    params['cookies']['sB'] = build_sb_cookie(sbcookie_params)
-
-    # Search region/language
-    domain = region2domain.get(region)
-    if not domain:
-        domain = lang2domain.get(lang, f'{lang}.search.yahoo.com')
-    logger.debug(f'domain selected: {domain}')
-    logger.debug(f'cookies: {params["cookies"]}')
-
-    params['url'] = f'https://{domain}/search?{urlencode(url_params)}'
-    params['domain'] = domain
 
 
-def parse_url(url_string):
+def request(query: str, params: "OnlineParams") -> None:
+    parts = params["searxng_locale"].split("-")
+    lang = yahoo_languages.get(parts[0], "any")
+
+    url_params: dict[str, str | int] = {"p": query}
+    if params["time_range"] in time_range_dict:
+        url_params["btf"] = time_range_dict[params["time_range"]]
+    if params["pageno"] == 1:
+        url_params["iscqry"] = ""
+    else:
+        url_params["b"] = params["pageno"] * 7 + 1
+        url_params["pz"] = 7
+        url_params["bct"] = 0
+        url_params["xargs"] = 0
+
+    params["cookies"]["sB"] = urlencode(
+        {
+            "v": 1,
+            "vm": safesearch_dict[params["safesearch"]],
+            "fl": 1,
+            "vl": f"lang_{lang}",
+            "pn": 10,
+            "rw": "new",
+            "userset": 1,
+        }
+    )
+
+    domain = "search.yahoo.com"
+    if len(parts) > 1 and parts[-1] in region2domain:
+        domain = region2domain[parts[-1]]
+    logger.debug("domain selected: %s", domain)
+    params["url"] = f"https://{domain}/search?{urlencode(url_params)}"
+    if ybv := CACHE.get("YBV"):
+        params["cookies"]["YBV"] = ybv
+
+
+def parse_url(url_string: str) -> str:
     """remove yahoo-specific tracking-url"""
 
-    endings = ['/RS', '/RK']
+    endings = ["/RS", "/RK"]
     endpositions = []
-    start = url_string.find('http', url_string.find('/RU=') + 1)
+    start = url_string.find("http", url_string.find("/RU=") + 1)
 
     for ending in endings:
         endpos = url_string.rfind(ending)
@@ -212,45 +181,64 @@ def parse_url(url_string):
     return unquote(url_string[start:end])
 
 
-def response(resp):
+def _yahoo_html(resp: "SXNG_Response") -> "SXNG_Response":
-    """parse response"""
+    cookies = dict(resp.search_params["cookies"])
+    params = resp.search_params
 
-    results = []
+    for _ in range(_YBV_HOPS):
+        if ybv := resp.cookies.get("YBV"):
+            cookies["YBV"] = ybv
+            if ybv.startswith("v0.2"):
+                CACHE.set("YBV", ybv, expire=86400)
+
+        if resp.status_code == 200:
+            return resp
+
+        loc = resp.headers.get("location")
+        if resp.status_code not in (302, 307) or not loc:
+            return resp
+
+        # request ourselves instead of following it
+        resp = get(
+            urljoin(resp.url, loc),
+            cookies=cookies,
+            headers=params["headers"],
+            allow_redirects=False,
+        )
+        resp.search_params = params
+
+    return resp
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    resp = _yahoo_html(resp)
+    results = EngineResults()
+    if resp.status_code != 200:
+        resp.raise_for_status()
     dom = html.fromstring(resp.text)
 
-    url_xpath = './/div[contains(@class,"compTitle")]/h3/a/@href'
-    title_xpath = './/h3//a/@aria-label'
-
-    domain = resp.search_params['domain']
-    if domain == "search.yahoo.com":
-        url_xpath = './/div[contains(@class,"compTitle")]/a/@href'
-        title_xpath = './/div[contains(@class,"compTitle")]/a/h3/span'
-
-    # parse results
     for result in eval_xpath_list(dom, '//div[contains(@class,"algo-sr")]'):
-        url = eval_xpath_getindex(result, url_xpath, 0, default=None)
+        url = eval_xpath_getindex(result, './/div[contains(@class,"compTitle")]//a/@href', 0, default=None)
         if url is None:
             continue
         url = parse_url(url)
-
+        title = extract_text(eval_xpath_getindex(result, ".//h3//a/@aria-label", 0, default=None), allow_none=True)
-        title = eval_xpath_getindex(result, title_xpath, 0, default='')
+        if not title:
-        title: str = extract_text(title)
+            title = extract_text(eval_xpath_getindex(result, ".//a/h3", 0, default=None), allow_none=True)
-        content = eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0, default='')
+        if not title:
-        content: str = extract_text(content, allow_none=True)
+            continue
-
+        content = extract_text(eval_xpath_getindex(result, './/div[contains(@class, "compText")]', 0))
-        # append result
+        results.add(
-        results.append(
+            results.types.MainResult(
-            {
+                url=url,
-                'url': url,
                 # title sometimes contains HTML tags / see
                 # https://github.com/searxng/searxng/issues/3790
-                'title': " ".join(html_to_text(title).strip().split()),
+                title=html_to_text(title),
-                'content': " ".join(html_to_text(content).strip().split()),
+                content=html_to_text(content),
-            }
+            )
         )
 
     for suggestion in eval_xpath_list(dom, '//div[contains(@class, "AlsoTry")]//table//a'):
-        # append suggestion
+        results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
-        results.append({'suggestion': extract_text(suggestion)})
 
     return results