searxng/searx/engines/bing.py

# SPDX-License-Identifier: AGPL-3.0-or-later
"""This is the implementation of the Bing-Web engine.  Some of this
implementations are shared by other engines:

- :ref:`bing images engine`
- :ref:`bing news engine`
- :ref:`bing videos engine`

.. note::

   Some functionality (paging and time-range results) are not supported
   since they depend on JavaScript.
"""

import base64
import typing as t
from urllib.parse import parse_qs, urlencode, urlparse

import babel
import babel.languages
from lxml import html

from searx.enginelib.traits import EngineTraits
from searx.locales import region_tag
from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text

if t.TYPE_CHECKING:
    from searx.extended_types import SXNG_Response
    from searx.search.processors import OnlineParams

about: dict[str, t.Any] = {
    "website": "https://www.bing.com",
    "wikidata_id": "Q182496",
    "official_api_documentation": "https://github.com/MicrosoftDocs/bing-docs",
    "use_official_api": False,
    "require_api_key": False,
    "results": "HTML",
}

# engine dependent config
categories = ["general", "web"]
safesearch = True
enable_http3 = True
_safesearch_map: dict[int, str] = {
    0: "off",
    1: "moderate",
    2: "strict",
}
"""Filter results. 0: None, 1: Moderate, 2: Strict"""

base_url = "https://www.bing.com"
"""Bing-Web search URL"""


def get_locale_params(engine_region: str | None) -> dict[str, str] | None:
    """API documentation states the ``mkt`` parameter is *the
    recommended primary signal* for locale:

        If known, you are encouraged to always specify the market.
        Specifying the market helps Bing route the request and return an
        appropriate and optimal response.

    The ``mkt`` parameter takes a full ``<language>-<country>`` code.

    This function is shared with :py:mod:`searx.engines.bing_images`,
    :py:mod:`searx.engines.bing_news`, and :py:mod:`searx.engines.bing_videos`.
    """

    if not engine_region or engine_region == "clear":
        return None

    return {"mkt": engine_region}


def request(query: str, params: "OnlineParams"):
    """Assemble a Bing-Web request."""

    engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)

    query_params: dict[str, str | int] = {
        "q": query,
        "adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
    }

    if engine_region and engine_region != "clear":
        lang, _, cc = engine_region.partition("-")
        query_params["setlang"] = lang
        if cc and cc not in ("us", "cn", "ru"):  # bing just sends junk for these
            query_params["cc"] = cc

    params["url"] = f"{base_url}/search?{urlencode(query_params)}"


def response(resp: "SXNG_Response") -> list[dict[str, t.Any]]:
    """Get response from Bing-Web"""

    results: list[dict[str, t.Any]] = []

    dom = html.fromstring(resp.text)

    for item in eval_xpath_list(dom, '//ol[@id="b_results"]/li[contains(@class, "b_algo")]'):
        link = eval_xpath_getindex(item, ".//h2/a", 0, None)
        if link is None:
            continue

        href = link.attrib.get("href", "")
        title = extract_text(link)

        if not href or not title:
            continue

        # what about cn.bing.com, ..?
        if href.startswith("https://www.bing.com/ck/a?"):
            qs = parse_qs(urlparse(href).query)
            u_values = qs.get("u")
            if u_values:
                u_val = u_values[0]
                if u_val.startswith("a1"):
                    encoded = u_val[2:]
                    # base64url without padding
                    encoded += "=" * (-len(encoded) % 4)
                    href = base64.urlsafe_b64decode(encoded).decode("utf-8", errors="replace")

        # remove decorative icons that Bing injects into <p> elements
        # (`<span class="algoSlug_icon">`)
        content_els = eval_xpath(item, ".//p")
        for p in content_els:
            for icon in p.xpath('.//span[@class="algoSlug_icon"]'):
                icon.getparent().remove(icon)
        content = extract_text(content_els)

        results.append({"url": href, "title": title, "content": content})

    return results


def fetch_traits(engine_traits: EngineTraits) -> None:
    """Fetch regions from Bing-Web."""
    # pylint: disable=import-outside-toplevel

    from searx.network import get  # see https://github.com/searxng/searxng/issues/762
    from searx.utils import gen_useragent

    headers = {
        "User-Agent": gen_useragent(),
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Language": "en-US;q=0.5,en;q=0.3",
        "DNT": "1",
        "Connection": "keep-alive",
        "Upgrade-Insecure-Requests": "1",
        "Sec-GPC": "1",
        "Cache-Control": "max-age=0",
    }

    resp = get("https://www.bing.com/account/general", headers=headers, timeout=5)
    if not resp.ok:
        raise RuntimeError("Response from Bing is not OK.")

    dom = html.fromstring(resp.text)

    map_market_codes: dict[str, str] = {
        "zh-hk": "en-hk",  # not sure why, but at Microslop this is the market code for Hongkong
    }

    for href in eval_xpath(dom, '//div[@id="region-section-content"]//div[@class="regionItem"]/a/@href'):
        cc_tag = parse_qs(urlparse(href).query)["cc"][0]
        if cc_tag == "clear":
            engine_traits.all_locale = cc_tag
            continue

        # add market codes from official languages of the country ..
        for lang_tag in babel.languages.get_official_languages(cc_tag, de_facto=True):
            lang_tag = lang_tag.split("_")[0]  # zh_Hant --> zh
            market_code = f"{lang_tag}-{cc_tag}"  # zh-tw
            market_code = map_market_codes.get(market_code, market_code)

            try:
                sxng_tag = region_tag(babel.Locale.parse("%s_%s" % (lang_tag, cc_tag.upper())))
            except babel.UnknownLocaleError:
                # silently ignore unknown languages
                continue

            conflict = engine_traits.regions.get(sxng_tag)
            if conflict:
                if conflict != market_code:
                    print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, market_code))
                continue

            engine_traits.regions[sxng_tag] = market_code