searxng/searx/engines/google_cse.py

# SPDX-License-Identifier: AGPL-3.0-or-later
"""Google Custom Search Engine"""

import datetime
import typing as t
from json import loads
from urllib.parse import urlencode

from searx.enginelib import EngineCache
from searx.exceptions import SearxEngineAPIException, SearxEngineTooManyRequestsException
from searx.network import get
from searx.result_types import EngineResults

from searx.engines.google import fetch_traits  # pylint: disable=unused-import
from searx.engines.google import filter_mapping, get_google_info

if t.TYPE_CHECKING:
    from searx.extended_types import SXNG_Response
    from searx.search.processors import OnlineParams

about = {
    "website": "https://www.google.com",
    "wikidata_id": "Q2233943",
    "official_api_documentation": "https://developers.google.com/custom-search/docs/element",
    "use_official_api": False,
    "require_api_key": False,
    "results": "JSONP",
    "description": "Platform for creating custom search engines based on Google Search.",
}

categories = ["general", "web"]
paging = True
max_page = 5
page_size = 20
time_range_support = True
language_support = True
safesearch = True

CX = "partner-pub-8993703457585266:4862972284"  # blackle.com

CACHE: EngineCache


def setup(engine_settings: dict[str, t.Any]) -> bool:
    global CACHE  # pylint: disable=global-statement
    CACHE = EngineCache(engine_settings["name"])
    return True


def _cse_token() -> dict[str, str]:
    token: dict[str, str] = CACHE.get(CX)
    if token:
        return token

    resp = get(f"https://www.google.com/cse/cse.js?cx={CX}", timeout=10)
    if not resp.ok:
        raise SearxEngineAPIException("failed to obtain cse token")

    end = resp.text.rfind("});")
    start = resp.text.rfind("({")
    opts: dict[str, str] = loads(resp.text[start + 1 : end + 1])

    cse_tok = opts.get("cse_token")
    if not cse_tok:
        raise SearxEngineAPIException("failed to obtain cse token")

    exp = opts.get("exp")
    token = {
        "cse_tok": cse_tok,
        "cselibv": opts.get("cselibVersion", ""),
        "exp": ",".join(exp) if exp else "",
    }
    CACHE.set(CX, token, expire=3600)
    return token


def _get_start_and_end_date_str(time_range: str) -> tuple[str, str]:
    time_range_map = {"day": 1, "week": 7, "month": 30, "year": 365}

    end_date = datetime.datetime.now()
    start_date = end_date - datetime.timedelta(days=time_range_map[time_range])

    return start_date.strftime("%Y%m%d"), end_date.strftime("%Y%m%d")


def request(query: str, params: "OnlineParams") -> None:
    token = _cse_token()

    google_info = get_google_info(params, traits)
    info: dict[str, str] = google_info["params"]

    hl = info["hl"]
    if params.get("searxng_locale", "all") == "all" or "ZZ" in hl:
        hl = "en"

    args = {
        "rsz": "filtered_cse",
        "num": str(page_size),
        "hl": hl,
        "cselibv": token["cselibv"],
        "cx": CX,
        "q": query,
        "safe": filter_mapping[params["safesearch"]],
        "cse_tok": token["cse_tok"],
        "callback": "_",
        "rurl": "",
    }
    if params["time_range"]:
        start_date, end_date = _get_start_and_end_date_str(params["time_range"])
        args["sort"] = f"date:r:{start_date}:{end_date}"

    if info.get("lr"):
        args["lr"] = info["lr"]
    if info.get("cr"):
        args["cr"] = info["cr"]
    if google_info["country"] not in (None, "ZZ"):
        args["gl"] = google_info["country"]
    if token["exp"]:
        args["exp"] = token["exp"]

    start = (params["pageno"] - 1) * page_size
    if start:
        args["start"] = str(start)

    params["url"] = "https://cse.google.com/cse/element/v1?" + urlencode(args)
    params["cookies"] = google_info["cookies"]
    params["headers"].update(google_info["headers"])
    params["headers"]["Referer"] = "https://cse.google.com/"


def response(resp: "SXNG_Response") -> EngineResults:
    json_resp = resp.text[resp.text.find("{") : resp.text.rfind("}") + 1]
    data = loads(json_resp)

    # not the real types, but a sufficient approximation
    item: dict[str, str]
    error: dict[str, str | int]

    if error := data.get("error"):
        message = error.get("message", "unknown error")
        if error.get("code") == 429:
            raise SearxEngineTooManyRequestsException(message=f"google cse: {message}")
        raise SearxEngineAPIException(f"google cse: {message}")

    results = EngineResults()
    for item in data.get("results", []):
        url = item.get("unescapedUrl")
        if not url:
            continue
        results.add(
            results.types.MainResult(
                url=url,
                title=item.get("titleNoFormatting", ""),
                content=item.get("contentNoFormatting", ""),
                thumbnail=item.get("richSnippet", {}).get("cseThumbnail", {}).get("src", ""),  # type: ignore
            )
        )
    return results