[fix] google: use Nokia UA (#6546)

searx/autocomplete.py

@@ -127,18 +127,17 @@ def duckduckgo(query: str, sxng_locale: str) -> list[str]:
 
 
 def google_complete(query: str, sxng_locale: str) -> list[str]:
-    """Autocomplete from Google.  Supports Google's languages and subdomains
+    """Autocomplete from Google.  Supports Google's languages
     (:py:obj:`searx.engines.google.get_google_info`) by using the async REST
     API::
 
-        https://{subdomain}/complete/search?{args}
+        https://www.google.com/complete/search?{args}
 
     """
 
     data = ENGINE_TRAITS.get("google") or {}
     traits = EngineTraits(**data)
     google_info: dict[str, t.Any] = google.get_google_info({'searxng_locale': sxng_locale}, traits)
-    url = 'https://{subdomain}/complete/search?{args}'
     args = urlencode(
         {
             'q': query,
@@ -148,7 +147,7 @@ def google_complete(query: str, sxng_locale: str) -> list[str]:
     )
     results: list[str] = []
 
-    resp = get(url.format(subdomain=google_info['subdomain'], args=args))
+    resp = get('https://www.google.com/complete/search?' + args)
     if resp and resp.ok:
         json_txt = resp.text[resp.text.find('[') : resp.text.find(']', -3) + 1]
         data = json.loads(json_txt)

searx/engines/google.py

@@ -9,12 +9,15 @@ engines:
 - :ref:`google scholar engine`
 - :ref:`google autocomplete`
 
+This implementation uses Nokia user agents to request an XML layout from Google.
+The normal web version requires executing JavaScript to load the results and
+therefore is currently not used here.  See `Google discussion`_ for more
+information on that topic.
+
+.. _Google discussion: https://github.com/searxng/searxng/issues/6359
 """
 
 import random
-import re
-import string
-import time
 import typing as t
 from urllib.parse import unquote, urlencode
 
@@ -44,16 +47,16 @@ about = {
     "official_api_documentation": "https://developers.google.com/custom-search/",
     "use_official_api": False,
     "require_api_key": False,
-    "results": "HTML",
+    "results": "XML",
 }
 
 # engine dependent config
 categories = ["general", "web"]
 paging = True
 max_page = 50
-"""`Google max 50 pages`_
+"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
 
-.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
+.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
 """
 time_range_support = True
 language_support = True
@@ -64,38 +67,23 @@ time_range_dict = {"day": "d", "week": "w", "month": "m", "year": "y"}
 # Filter results. 0: None, 1: Moderate, 2: Strict
 filter_mapping = {0: "off", 1: "medium", 2: "high"}
 
+# https://github.com/searxng/searxng/issues/6359
+nokia_useragents = (
+    "Nokia7610/2.0 (5.0509.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
+    "Nokia7610/2.0 (7.0642.0) SymbianOS/7.0s Series60/2.1 Profile/MIDP-2.0 Configuration/CLDC-1.0",
+    "Nokia6230/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Nokia6230i/2.0 (03.80) Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "Nokia6280/2.0 (03.60) Profile/MIDP-2.0 Configuration/CLDC-1.1",
+    "NokiaN72/2.0617.1.0.3 Series60/2.8 Profile/MIDP-2.0 Configuration/CLDC-1.1",
+)
+
+
 # specific xpath variables
 # ------------------------
 
 # Suggestions are links placed in a *card-section*, we extract only the text
 # from the links not the links itself.
-suggestion_xpath = '//div[contains(@class, "gGQDvd iIWm4b")]//a'
+suggestion_xpath = '//table[contains(@class, "HExoMb")]//a[contains(@class, "ZWRArf")]'
-
-
-_arcid_range = string.ascii_letters + string.digits + "_-"
-_arcid_random: tuple[str, int] | None = None
-
-
-def ui_async(start: int) -> str:
-    """Format of the response from UI's async request.
-
-    - ``arc_id:<...>,use_ac:true,_fmt:prog``
-
-    The arc_id is random generated every hour.
-    """
-    global _arcid_random  # pylint: disable=global-statement
-
-    use_ac = "use_ac:true"
-    # _fmt:html returns a HTTP 500 when user search for celebrities like
-    # '!google natasha allegri' or '!google chris evans'
-    _fmt = "_fmt:prog"
-
-    # create a new random arc_id every hour
-    if not _arcid_random or (int(time.time()) - _arcid_random[1]) > 3600:
-        _arcid_random = ("".join(random.choices(_arcid_range, k=23)), int(time.time()))
-    arc_id = f"arc_id:srp_{_arcid_random[0]}_1{start:02}"
-
-    return ",".join([arc_id, use_ac, _fmt])
 
 
 def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[str, t.Any]:
@@ -127,19 +115,11 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
             A instance of :py:obj:`babel.core.Locale` build from the
             ``searxng_locale`` value.
 
-        subdomain:
-            Google subdomain :py:obj:`google_domains` that fits to the country
-            code.
-
         params:
             Py-Dictionary with additional request arguments (can be passed to
             :py:func:`urllib.parse.urlencode`).
 
             - ``hl`` parameter: specifies the interface language of user interface.
-            - ``lr`` parameter: restricts search results to documents written in
-              a particular language.
-            - ``cr`` parameter: restricts search results to documents
-              originating in a particular country.
             - ``ie`` parameter: sets the character encoding scheme that should
               be used to interpret the query string ('utf8').
             - ``oe`` parameter: sets the character encoding scheme that should
@@ -156,7 +136,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     ret_val: dict[str, t.Any] = {
         "language": None,
         "country": None,
-        "subdomain": None,
         "params": {},
         "headers": {},
         "cookies": {},
@@ -169,7 +148,7 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     except babel.core.UnknownLocaleError:
         locale = None
 
-    eng_lang = eng_traits.get_language(sxng_locale, "lang_en")
+    eng_lang = eng_traits.get_language(sxng_locale) or "lang_en"
     lang_code = eng_lang.split("_")[-1]  # lang_zh-TW --> zh-TW / lang_en --> en
     country = eng_traits.get_region(sxng_locale, eng_traits.all_locale)
 
@@ -184,7 +163,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     ret_val["language"] = eng_lang
     ret_val["country"] = country
     ret_val["locale"] = locale
-    ret_val["subdomain"] = eng_traits.custom["supported_domains"].get(country.upper(), "www.google.com")
 
     # hl parameter:
     #   The hl parameter specifies the interface language (host language) of
@@ -223,6 +201,8 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
 
     # specify a region (country) only if a region is given in the selected
     # locale --> https://github.com/searxng/searxng/issues/2672
+
+    if country is not None:
         ret_val["params"]["cr"] = ""
         if len(sxng_locale.split("-")) > 1:
             ret_val["params"]["cr"] = "country" + country
@@ -300,88 +280,77 @@ def detect_google_sorry(resp: "SXNG_Response"):
         raise SearxEngineCaptchaException()
 
 
-def request(query: str, params: "OnlineParams") -> None:
+def unwrap_google_url(raw_url: str) -> str:
-    """Google search request"""
+    # remove redirector from url
-    # pylint: disable=line-too-long
+    if raw_url.startswith("/url?q="):
-    start = (params["pageno"] - 1) * 10
+        return unquote(raw_url[7:].split("&sa=U")[0])
-    google_info = get_google_info(params, traits)
+    return raw_url
-
-    # https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium
-    query_url = (
-        "https://"
-        + google_info["subdomain"]
-        + "/search"
-        + "?"
-        + urlencode(
-            {
-                "q": query,
-                **google_info["params"],
-                "filter": "0",
-                "start": start,
-                # 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',
-                # 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',
-                # 'cs' : 1,
-                # 'sa': 'N',
-                # 'yv': 3,
-                # 'prmd': 'vin',
-                # 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',
-                # 'sa': 'N',
-                # 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'
-                # formally known as use_mobile_ui
-                # "asearch": "arc",
-                # "async": str_async,
-            }
-        )
-    )
-
-    if params["time_range"] in time_range_dict:
-        query_url += "&" + urlencode({"tbs": "qdr:" + time_range_dict[params["time_range"]]})
-    if params["safesearch"]:
-        query_url += "&" + urlencode({"safe": filter_mapping[params["safesearch"]]})
-    params["url"] = query_url
-
-    params["cookies"] = google_info["cookies"]
-    params["headers"].update(google_info["headers"])
 
 
-# regex match to get image map that is found inside the returned javascript:
+def wml_dom(resp: "SXNG_Response"):
-# (function(){var s='...';var i=['...'] ...}
-RE_DATA_IMAGE = re.compile(r"(data:image[^']*?)'[^']*?'((?:dimg|pimg|tsuid)[^']*)")
-
-
-def parse_url_images(text: str):
-    data_image_map = {}
-
-    for image_url, img_id in RE_DATA_IMAGE.findall(text):
-        data_image_map[img_id] = image_url.encode('utf-8').decode("unicode-escape")
-    logger.debug("data:image objects --> %s", list(data_image_map.keys()))
-    return data_image_map
-
-
-def response(resp: "SXNG_Response"):
-    """Get response from google's search request"""
-    # pylint: disable=too-many-branches, too-many-statements
     detect_google_sorry(resp)
-    data_image_map = parse_url_images(resp.text)
+    text = resp.text
+    if text.lstrip().startswith("<?xml"):
+        text = text.split("?>", 1)[-1]
+    return html.fromstring(text)
 
+
+def google_request(
+    query: str,
+    params: "OnlineParams",
+    extra_args: dict[str, t.Any] | None = None,
+    *,
+    eng_traits: EngineTraits | None = None,
+    use_time_range: bool = True,
+    use_safesearch: bool = True,
+    safesearch_map: dict[int, str] | None = None,
+    use_locales: bool = True,
+) -> None:
+    google_info = get_google_info(params, eng_traits or traits)
+    if not use_locales:
+        google_info["params"].pop("lr")
+        google_info["params"].pop("cr")
+
+    start = (params["pageno"] - 1) * 10
+    args: dict[str, t.Any] = {
+        "q": query,
+        "sca_esv": "1",
+        **google_info["params"],
+        **(extra_args or {}),
+    }
+    if start:
+        args["start"] = start
+    if use_time_range and params["time_range"] in time_range_dict:
+        args["tbs"] = "qdr:" + time_range_dict[params["time_range"]]
+    if use_safesearch and params["safesearch"]:
+        args["safe"] = (safesearch_map or filter_mapping)[params["safesearch"]]
+
+    params["url"] = f"https://www.google.com/wml/search?{urlencode(args)}"
+    params["headers"]["User-Agent"] = random.choice(nokia_useragents)
+
+
+def request(query: str, params: "OnlineParams") -> None:
+    google_request(query, params)
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
     results = EngineResults()
-
+    dom = wml_dom(resp)
-    # convert the text to dom
-    dom = html.fromstring(resp.text)
 
     # parse results
-    for result in eval_xpath_list(dom, '//a[@data-ved and not(@class)]'):
+    for result in eval_xpath_list(dom, '//div[contains(@class, "zMzFAb")]'):
-        # pylint: disable=too-many-nested-blocks
 
         try:
-            title_tag = eval_xpath_getindex(result, './/div[@style]', 0, default=None)
+            title_tag = eval_xpath_getindex(
+                result, './/a[contains(@class, "fuLhoc")]//span[contains(@class, "CVA68e")]', 0, default=None
+            )
             if title_tag is None:
                 # this not one of the common google results *section*
                 logger.debug("ignoring item from the result_xpath list: missing title")
                 continue
             title = extract_text(title_tag)
 
-            raw_url = result.get("href")
+            raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
             if raw_url is None:
                 logger.debug(
                     'ignoring item from the result_xpath list: missing url of title "%s"',
@@ -389,30 +358,19 @@ def response(resp: "SXNG_Response"):
                 )
                 continue
 
-            if raw_url.startswith('/url?q='):
+            url = unwrap_google_url(raw_url)
-                url = unquote(raw_url[7:].split("&sa=U")[0])  # remove the google redirector
+            content = extract_text(
-            else:
+                eval_xpath(result, './/div[contains(@class, "taTFJ")]//span[contains(@class, "FrIlee")]')
-                url = raw_url
+            )
-
+            thumbnail = eval_xpath_getindex(result, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
-            content_nodes = eval_xpath(result, '../..//div[contains(@class, "ilUpNd H66NU aSRlid")]')
+            results.add(
-            for item in content_nodes:
+                results.types.MainResult(
-                for script in item.xpath(".//script"):
+                    url=url,
-                    script.getparent().remove(script)
+                    title=title or "",
-
+                    content=content or "",
-            content = extract_text(content_nodes[0])
+                    thumbnail=thumbnail or "",
-
+                )
-            # Images that are NOT the favicon
+            )
-            xpath_image = eval_xpath_getindex(result, './/img', index=0, default=None)
-
-            thumbnail = None
-            if xpath_image is not None:
-                thumbnail = xpath_image.get("src")
-                if thumbnail.startswith("data:image"):
-                    img_id = xpath_image.get("id")
-                    if img_id:
-                        thumbnail = data_image_map.get(img_id)
-
-            results.append({"url": url, "title": title, "content": content or '', "thumbnail": thumbnail})
 
         except Exception as e:  # pylint: disable=broad-except
             logger.error(e, exc_info=True)
@@ -420,10 +378,8 @@ def response(resp: "SXNG_Response"):
 
     # parse suggestion
     for suggestion in eval_xpath_list(dom, suggestion_xpath):
-        # append suggestion
+        results.add(results.types.LegacyResult(suggestion=extract_text(suggestion)))
-        results.append({"suggestion": extract_text(suggestion)})
 
-    # return results
     return results
 
 
@@ -456,14 +412,12 @@ skip_countries = [
 ]
 
 
-def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
+def fetch_traits(engine_traits: EngineTraits):
     """Fetch languages from Google."""
     # pylint: disable=import-outside-toplevel, too-many-branches
 
     from searx.network import get  # see https://github.com/searxng/searxng/issues/762
 
-    engine_traits.custom["supported_domains"] = {}
-
     resp = get("https://www.google.com/preferences", timeout=5)
     if not resp.ok:
         raise RuntimeError("Response from Google preferences is not OK.")
@@ -514,22 +468,3 @@ def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True):
 
     # alias regions
     engine_traits.regions["zh-CN"] = "HK"
-
-    # supported domains
-
-    if add_domains:
-        resp = get("https://www.google.com/supported_domains", timeout=5)
-        if not resp.ok:
-            raise RuntimeError("Response from Google supported domains is not OK.")
-
-        for domain in resp.text.split():
-            domain = domain.strip()
-            if not domain or domain in [
-                ".google.com",
-            ]:
-                continue
-            region = domain.split(".")[-1].upper()
-            engine_traits.custom["supported_domains"][region] = "www" + domain
-            if region == "HK":
-                # There is no google.cn, we use .com.hk for zh-CN
-                engine_traits.custom["supported_domains"]["CN"] = "www" + domain

searx/engines/google_cse.py

@@ -95,12 +95,11 @@ def request(query: str, params: "OnlineParams") -> None:
     token = _cse_token()
 
     google_info = get_google_info(params, traits)
-    info: dict[str, str] = google_info["params"]
 
     args = {
         "rsz": "filtered_cse",
         "num": str(page_size),
-        "hl": info["hl"],
+        "hl": google_info["params"]["hl"],
         "cselibv": token["cselibv"],
         "cx": CX,
         "q": query,
@@ -114,10 +113,6 @@ def request(query: str, params: "OnlineParams") -> None:
         start_date, end_date = _get_start_and_end_date_str(params["time_range"])
         args["sort"] = f"date:r:{start_date}:{end_date}"
 
-    if info.get("lr"):
-        args["lr"] = info["lr"]
-    if info.get("cr"):
-        args["cr"] = info["cr"]
     if google_info["country"] not in (None, "ZZ"):
         args["gl"] = google_info["country"]
     if token["exp"]:

searx/engines/google_images.py

@@ -1,122 +1,75 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""This is the implementation of the Google Images engine using the internal
+"""Google Images: see :py:obj:`searx.engines.google`."""
-Google API used by the Google Go Android app.
 
-This internal API offer results in
+import typing as t
-
+from urllib.parse import parse_qs, unquote, urlparse
-- JSON (``_fmt:json``)
-- Protobuf_ (``_fmt:pb``)
-- Protobuf_ compressed? (``_fmt:pc``)
-- HTML (``_fmt:html``)
-- Protobuf_ encoded in JSON (``_fmt:jspb``).
-
-.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers
-"""
-
-from urllib.parse import urlencode
-from json import loads
 
 from searx.engines.google import fetch_traits  # pylint: disable=unused-import
-from searx.engines.google import (
+from searx.engines.google import google_request, wml_dom
-    get_google_info,
+from searx.result_types import EngineResults
-    time_range_dict,
+from searx.utils import eval_xpath_list
-    detect_google_sorry,
+
-)
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
 
 # about
 about = {
-    "website": 'https://images.google.com',
+    "website": "https://images.google.com",
-    "wikidata_id": 'Q521550',
+    "wikidata_id": "Q521550",
-    "official_api_documentation": 'https://developers.google.com/custom-search',
+    "official_api_documentation": "https://developers.google.com/custom-search",
     "use_official_api": False,
     "require_api_key": False,
-    "results": 'JSON',
+    "results": "XML",
 }
 
 # engine dependent config
-categories = ['images', 'web']
+categories = ["images", "web"]
 paging = True
 max_page = 50
-"""`Google max 50 pages`_
+"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
 
-.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982
+.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
 """
 
 time_range_support = True
 language_support = True
 safesearch = True
 
-filter_mapping = {0: 'images', 1: 'active', 2: 'active'}
+filter_mapping = {0: "images", 1: "active", 2: "active"}
 
 
-def request(query, params):
+def request(query: str, params: "OnlineParams") -> None:
-    """Google-Image search request"""
+    google_request(
-
+        query,
-    google_info = get_google_info(params, traits)
+        params,
-
+        {"tbm": "isch"},
-    query_url = (
+        eng_traits=traits,
-        'https://'
+        safesearch_map=filter_mapping,
-        + google_info['subdomain']
+        use_locales=False,
-        + '/search'
-        + '?'
-        + urlencode({'q': query, 'tbm': "isch", **google_info['params'], 'asearch': 'isch'})
-        # don't urlencode this because wildly different AND bad results
-        # pagination uses Zero-based numbering
-        + f'&async=_fmt:json,p:1,ijn:{params["pageno"] - 1}'
     )
 
-    if params['time_range'] in time_range_dict:
+
-        query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
+def response(resp: "SXNG_Response") -> EngineResults:
-    if params['safesearch']:
+    results = EngineResults()
-        query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
+    dom = wml_dom(resp)
-    params['url'] = query_url
+
-    params['cookies'] = google_info['cookies']
+    for link in eval_xpath_list(dom, '//a[contains(@href, "/imgres?")]'):
-    params['headers'].update(google_info['headers'])
+        qs = parse_qs(urlparse(link.get("href", "")).query)
-    # this ua will allow getting ~50 results instead of 10. #1641
+        img_src = qs.get("imgurl", [""])[0]
-    params['headers']['User-Agent'] = (
+        url = qs.get("imgrefurl", [""])[0]
-        'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;' f' {google_info.get("country", "US")}) gzip'
+        if not img_src or not url:
+            continue
+        width, height = qs.get("w", [""])[0], qs.get("h", [""])[0]
+        tbnid = qs.get("tbnid", [""])[0]
+        results.add(
+            results.types.Image(
+                url=url,
+                title=unquote(urlparse(img_src).path.rsplit("/", 1)[-1]) or urlparse(url).netloc,
+                img_src=img_src,
+                thumbnail_src=f"https://encrypted-tbn0.gstatic.com/images?q=tbn:{tbnid}",
+                resolution=f"{width} x {height}" if width and height else "",
+            )
         )
-
-    return params
-
-
-def response(resp):
-    """Get response from google's search request"""
-    results = []
-
-    detect_google_sorry(resp)
-
-    json_start = resp.text.find('{"ischj":')
-    json_data = loads(resp.text[json_start:])
-
-    for item in json_data["ischj"].get("metadata", []):
-        result_item = {
-            'url': item["result"]["referrer_url"],
-            'title': item["result"]["page_title"],
-            'content': item["text_in_grid"]["snippet"],
-            'source': item["result"]["site_title"],
-            'resolution': f'{item["original_image"]["width"]} x {item["original_image"]["height"]}',
-            'img_src': item["original_image"]["url"],
-            'thumbnail_src': item["thumbnail"]["url"],
-            'template': 'images.html',
-        }
-
-        author = item["result"].get('iptc', {}).get('creator')
-        if author:
-            result_item['author'] = ', '.join(author)
-
-        copyright_notice = item["result"].get('iptc', {}).get('copyright_notice')
-        if copyright_notice:
-            result_item['source'] += ' | ' + copyright_notice
-
-        freshness_date = item["result"].get("freshness_date")
-        if freshness_date:
-            result_item['source'] += ' | ' + freshness_date
-
-        file_size = item.get('gsa', {}).get('file_size')
-        if file_size:
-            result_item['source'] += ' (%s)' % file_size
-
-        results.append(result_item)
 
     return results

searx/engines/google_news.py

@@ -1,324 +1,91 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""This is the implementation of the Google News engine.
+"""Google News: see :py:obj:`searx.engines.google`."""
 
-Google News has a different region handling compared to Google WEB.
-
-- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)
-- the hl_ argument has to be set correctly (and different to Google WEB)
-- the gl_ argument is mandatory
-
-If one of this argument is not set correctly, the request is redirected to
-CONSENT dialog::
-
-  https://consent.google.com/m?continue=
-
-The google news API ignores some parameters from the common :ref:`google API`:
-
-- num_ : the number of search results is ignored / there is no paging all
-  results for a query term are in the first response.
-- save_ : is ignored / Google-News results are always *SafeSearch*
-
-.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp
-.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp
-.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp
-.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp
-"""
 import typing as t
 
-import json
+from searx.engines.google import fetch_traits  # pylint: disable=unused-import
-import base64
+from searx.engines.google import google_request, unwrap_google_url, wml_dom
-from urllib.parse import urlencode
+from searx.result_types import EngineResults
-from lxml import html
-import babel
-
-from searx import locales
 from searx.utils import (
-    eval_xpath,
-    eval_xpath_list,
     eval_xpath_getindex,
+    eval_xpath_list,
     extract_text,
 )
 
-from searx.engines.google import fetch_traits as _fetch_traits  # pylint: disable=unused-import
-from searx.engines.google import (
-    get_google_info,
-    detect_google_sorry,
-)
-from searx.enginelib.traits import EngineTraits
-
-from searx.result_types import EngineResults
-
 if t.TYPE_CHECKING:
     from searx.extended_types import SXNG_Response
     from searx.search.processors import OnlineParams
 
 # about
 about = {
-    "website": "https://news.google.com",
+    "website": "https://www.google.com",
     "wikidata_id": "Q12020",
     "official_api_documentation": "https://developers.google.com/custom-search",
     "use_official_api": False,
     "require_api_key": False,
-    "results": "HTML",
+    "results": "XML",
 }
 
 # engine dependent config
 categories = ["news"]
-paging = False
+paging = True
+max_page = 50
+"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
+
+.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
+"""
 time_range_support = False
 language_support = True
-
+safesearch = False
-# Google-News results are always *SafeSearch*. Option 'safesearch' is set to
-# False here.
-#
-#  safesearch : results are identical for safesearch=0 and safesearch=2
-safesearch = True
-base_url: str = "https://news.google.com"
 
 
 def request(query: str, params: "OnlineParams") -> None:
-    """Google-News search request"""
+    google_request(
-
+        query,
-    sxng_locale = params.get("searxng_locale", "en-US")
+        params,
-    ceid: str = locales.get_engine_locale(
+        {"tbm": "nws"},
-        sxng_locale, traits.custom["ceid"], default="US:en"
+        eng_traits=traits,
-    )  # pyright: ignore[reportAssignmentType]
+        use_time_range=False,
-    google_info = get_google_info(params, traits)
+        use_safesearch=False,
-    google_info["subdomain"] = "news.google.com"  # google news has only one domain
+        use_locales=False,
-
-    ceid_region, ceid_lang = ceid.split(":")
-    ceid_lang, ceid_suffix = (
-        ceid_lang.split(":")
-        + [
-            "",
-        ]
-    )[:2]
-
-    google_info["params"]["hl"] = ceid_lang
-
-    if ceid_suffix and ceid_suffix not in ["Hans", "Hant"]:
-
-        if ceid_region.lower() == ceid_lang:
-            google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
-        else:
-            google_info["params"]["hl"] = ceid_lang + "-" + ceid_suffix
-
-    elif ceid_region.lower() != ceid_lang:
-
-        if ceid_region in ["AT", "BE", "CH", "IL", "SA", "IN", "BD", "PT"]:
-            google_info["params"]["hl"] = ceid_lang
-        else:
-            google_info["params"]["hl"] = ceid_lang + "-" + ceid_region
-
-    google_info["params"]["lr"] = "lang_" + ceid_lang.split("-")[0]
-    google_info["params"]["gl"] = ceid_region
-
-    query_url = (
-        "https://"
-        + google_info["subdomain"]
-        + "/search?"
-        + urlencode(
-            {"q": query, **google_info["params"]},
-        )
-        # ceid includes a ':' character which must not be urlencoded
-        + ("&ceid=%s" % ceid)
     )
 
-    params["url"] = query_url
+
-    params["cookies"] = google_info["cookies"]
+def _span_text(link, css_class: str):
-    params["headers"].update(google_info["headers"])
+    return extract_text(
+        eval_xpath_getindex(link, f'.//span[contains(@class, "{css_class}")]', 0, default=None),
+        allow_none=True,
+    )
 
 
 def response(resp: "SXNG_Response") -> EngineResults:
-    """Get response from google's search request"""
+    results = EngineResults()
-
+    seen = set()
-    res = EngineResults()
+    for link in eval_xpath_list(wml_dom(resp), '//a[contains(@href, "/url?q=")]'):
-
+        href = link.get("href")
-    detect_google_sorry(resp)
+        if not href:
-
-    # convert the text to dom
-    dom = html.fromstring(resp.text)
-
-    for result in eval_xpath_list(dom, "//div[@jslog and @data-n-tid and @jsdata]"):
-
-        url: str = eval_xpath_getindex(result, "./a[@target='_blank']/@href", 0, default=0)
-        if not url:
-            continue
-        if url.startswith("./"):
-            url = base_url + url[1:]
-
-        # The real URL is often encoded in the "jslog" attribute
-        jslog: str | None = eval_xpath_getindex(result, "./a[@target='_blank']/@jslog", 0, default=None)
-
-        # Try to extract the real URL from jslog
-        real_url: str | None = None
-        if jslog:
-            # jslog format is usually: "95014; 5:<base64>; track:click,vis".  We
-            # want the second part (index 1) after splitting by ";"
-            parts: list[str] = jslog.split(";")
-            if len(parts) > 1:
-                b64_data: str = parts[1].split(":")[-1].strip()
-                # Pad base64 if necessary
-                b64_data += "=" * (-len(b64_data) % 4)
-                decoded_data: list[str | None] = json.loads(base64.b64decode(b64_data).decode("utf-8"))
-                # The URL is typically the last element in the decoded array
-                if (
-                    isinstance(decoded_data, list)
-                    and isinstance(decoded_data[-1], str)
-                    and decoded_data[-1].startswith("http")
-                ):
-                    real_url = decoded_data[-1]
-        if real_url:
-            url = real_url
-        else:
-            logger.error(f"no real-url found: {url}")
             continue
 
-        title = extract_text(eval_xpath(result, "./h4")) or ""
+        url = unwrap_google_url(href)
+        if url in seen or "google.com/search" in url:
+            continue
 
-        # The pub_date is mostly a string like 'yesterday', not a real timezone
+        title = _span_text(link, "M3vVJe") or _span_text(link, "fuLhoc")
-        # date or time.  Therefore we can't use publishedDate and place the
+        if not title:
-        # *pub* sting into the content.
+            continue
 
-        pub_date = extract_text(eval_xpath(result, ".//time"))
+        source = _span_text(link, "dXDvrc")
-        pub_origin = extract_text(eval_xpath(result, ".//div[contains(@class, 'vr1PYe')]"))
+        pub_date = _span_text(link, "YVIcad")
-        content = " / ".join([x for x in [pub_origin, pub_date] if x])
+        thumbnail = eval_xpath_getindex(link, './/img[contains(@src, "encrypted-tbn")]/@src', 0, default=None)
 
-        thumbnail: str = eval_xpath_getindex(result, ".//figure/img/@src", 0, default="")
+        seen.add(url)
-        if thumbnail and thumbnail.startswith("/"):
+        results.add(
-            thumbnail = base_url + thumbnail
+            results.types.MainResult(
-
-        res.add(
-            res.types.MainResult(
                 url=url,
                 title=title,
-                content=content,
+                content=" / ".join(x for x in [source, pub_date] if x),
-                thumbnail=thumbnail,
+                thumbnail=thumbnail or "",
             )
         )
 
-    return res
+    return results
-
-
-ceid_list = [
-    "AE:ar",
-    "AR:es-419",
-    "AT:de",
-    "AU:en",
-    "BD:bn",
-    "BE:fr",
-    "BE:nl",
-    "BG:bg",
-    "BR:pt-419",
-    "BW:en",
-    "CA:en",
-    "CA:fr",
-    "CH:de",
-    "CH:fr",
-    "CL:es-419",
-    "CN:zh-Hans",
-    "CO:es-419",
-    "CU:es-419",
-    "CZ:cs",
-    "DE:de",
-    "EE:et",
-    "EG:ar",
-    "ES:ca",
-    "ES:es",
-    "ET:en",
-    "FI:fi",
-    "FR:fr",
-    "GB:en",
-    "GH:en",
-    "GR:el",
-    "HK:zh-Hant",
-    "HU:hu",
-    "ID:en",
-    "ID:id",
-    "IE:en",
-    "IL:en",
-    "IL:he",
-    "IN:bn",
-    "IN:en",
-    "IN:gu",
-    "IN:hi",
-    "IN:ml",
-    "IN:mr",
-    "IN:pa",
-    "IN:ta",
-    "IN:te",
-    "IT:it",
-    "JP:ja",
-    "KE:en",
-    "KR:ko",
-    "LB:ar",
-    "LT:lt",
-    "LV:en",
-    "LV:lv",
-    "MA:fr",
-    "MY:en",
-    "MY:ms",
-    "NA:en",
-    "NG:en",
-    "NL:nl",
-    "NO:no",
-    "NZ:en",
-    "PH:en",
-    "PK:en",
-    "PL:pl",
-    "RO:ro",
-    "RS:sr",
-    "RU:ru",
-    "SA:ar",
-    "SE:sv",
-    "SG:en",
-    "SI:sl",
-    "SK:sk",
-    "SN:fr",
-    "TH:th",
-    "TR:tr",
-    "TZ:en",
-    "UA:ru",
-    "UA:uk",
-    "UG:en",
-    "US:en",
-    "VN:vi",
-    "ZA:en",
-    "ZW:en",
-]
-"""List of region/language combinations supported by Google News.  Values of the
-``ceid`` argument of the Google News REST API."""
-
-
-_skip_values = [
-    "ET:en",  # english (ethiopia)
-    "ID:en",  # english (indonesia)
-    "LV:en",  # english (latvia)
-]
-
-_ceid_locale_map = {"NO:no": "nb-NO"}
-
-
-def fetch_traits(engine_traits: EngineTraits):
-    _fetch_traits(engine_traits, add_domains=False)
-
-    engine_traits.custom["ceid"] = {}
-
-    for ceid in ceid_list:
-        if ceid in _skip_values:
-            continue
-
-        region, lang = ceid.split(":")
-        x = lang.split("-")
-        if len(x) > 1:
-            if x[1] not in ["Hant", "Hans"]:
-                lang = x[0]
-
-        sxng_locale = _ceid_locale_map.get(ceid, lang + "-" + region)
-        try:
-            locale = babel.Locale.parse(sxng_locale, sep="-")
-        except babel.UnknownLocaleError:
-            print("ERROR: %s -> %s is unknown by babel" % (ceid, sxng_locale))
-            continue
-
-        engine_traits.custom["ceid"][locales.region_tag(locale)] = ceid

searx/engines/google_scholar.py

@@ -77,8 +77,6 @@ def request(query: str, params: "OnlineParams") -> None:
     """Google-Scholar search request"""
 
     google_info = get_google_info(params, traits)
-    # subdomain is: scholar.google.xy
-    google_info["subdomain"] = google_info["subdomain"].replace("www.", "scholar.")
 
     args = {
         "q": query,
@@ -89,7 +87,7 @@ def request(query: str, params: "OnlineParams") -> None:
     }
     args.update(time_range_args(params))
 
-    params["url"] = "https://" + google_info["subdomain"] + "/scholar?" + urlencode(args)
+    params["url"] = "https://scholar.google.com/scholar?" + urlencode(args)
     params["cookies"] = google_info["cookies"]
     params["headers"].update(google_info["headers"])
 

searx/engines/google_videos.py

@@ -1,185 +1,87 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""This is the implementation of the Google Videos engine.
+"""Google Videos: see :py:obj:`searx.engines.google`."""
 
-.. admonition:: Content-Security-Policy (CSP)
+import typing as t
-
-   This engine needs to allow images from the `data URLs`_ (prefixed with the
-   ``data:`` scheme)::
-
-     Header set Content-Security-Policy "img-src 'self' data: ;"
-
-.. _data URLs:
-   https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
-"""
-import re
-from urllib.parse import urlencode, urlparse, parse_qs, unquote
-from lxml import html
-
-from searx.utils import (
-    eval_xpath_list,
-    eval_xpath_getindex,
-    extract_text,
-)
 
 from searx.engines.google import fetch_traits  # pylint: disable=unused-import
-from searx.engines.google import (
+from searx.engines.google import google_request, unwrap_google_url, wml_dom
-    get_google_info,
+from searx.result_types import EngineResults
-    time_range_dict,
+from searx.utils import (
-    filter_mapping,
+    eval_xpath_getindex,
-    suggestion_xpath,
+    eval_xpath_list,
-    detect_google_sorry,
+    extract_text,
-    ui_async,
+    get_embeded_stream_url,
+    parse_duration_string,
 )
-from searx.utils import get_embeded_stream_url
+
+if t.TYPE_CHECKING:
+    from searx.extended_types import SXNG_Response
+    from searx.search.processors import OnlineParams
 
 # about
 about = {
-    "website": 'https://www.google.com',
+    "website": "https://www.google.com",
-    "wikidata_id": 'Q219885',
+    "wikidata_id": "Q219885",
-    "official_api_documentation": 'https://developers.google.com/custom-search',
+    "official_api_documentation": "https://developers.google.com/custom-search",
     "use_official_api": False,
     "require_api_key": False,
-    "results": 'HTML',
+    "results": "XML",
 }
 
 # engine dependent config
-categories = ['videos', 'web']
+categories = ["videos", "web"]
 paging = True
 max_page = 50
+"""Google supports up to 50 pages of results, see the `Google max_page discussion`_.
+
+.. _Google max_page discussion: https://github.com/searxng/searxng/issues/2982
+"""
 language_support = True
 time_range_support = True
 safesearch = True
 
 
-# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA
+def request(query: str, params: "OnlineParams") -> None:
-# ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;
+    google_request(
-RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);?')
+        query,
-
+        params,
-
+        {"tbm": "vid"},
-def parse_data_images(text: str):
+        eng_traits=traits,
-    data_image_map = {}
+        use_locales=False,
-
-    for img_id, data_image in RE_DATA_IMAGE.findall(text):
-        end_pos = data_image.rfind("=")
-        if end_pos > 0:
-            data_image = data_image[: end_pos + 1]
-        data_image_map[img_id] = data_image
-    logger.debug("data:image objects --> %s", list(data_image_map.keys()))
-    return data_image_map
-
-
-def request(query, params):
-    """Google-Video search request"""
-    google_info = get_google_info(params, traits)
-    start = (params['pageno'] - 1) * 10
-
-    query_url = (
-        'https://'
-        + google_info['subdomain']
-        + '/search'
-        + "?"
-        + urlencode(
-            {
-                'q': query,
-                'tbm': "vid",
-                'start': start,
-                **google_info['params'],
-                'asearch': 'arc',
-                'async': ui_async(start),
-            }
-        )
     )
 
-    if params['time_range'] in time_range_dict:
-        query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})
-    if 'safesearch' in params:
-        query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})
-    params['url'] = query_url
 
-    params['cookies'] = google_info['cookies']
+def response(resp: "SXNG_Response") -> EngineResults:
-    params['headers'].update(google_info['headers'])
+    results = EngineResults()
-    return params
 
-
+    for result in eval_xpath_list(wml_dom(resp), '//div[contains(@class, "zMzFAb")]'):
-def response(resp):
-    """Get response from google's search request"""
-    results = []
-
-    detect_google_sorry(resp)
-    data_image_map = parse_data_images(resp.text)
-
-    # convert the text to dom
-    dom = html.fromstring(resp.text)
-
-    result_divs = eval_xpath_list(dom, '//div[contains(@class, "MjjYud")]')
-
-    # parse results
-    for result in result_divs:
         title = extract_text(
-            eval_xpath_getindex(result, './/h3[contains(@class, "LC20lb")] | .//div[@role="heading"]', 0, default=None),
+            eval_xpath_getindex(result, './/span[contains(@class, "CVA68e")]', 0, default=None),
             allow_none=True,
         )
-        url = eval_xpath_getindex(
+        raw_url = eval_xpath_getindex(result, './/a[contains(@class, "fuLhoc")]/@href', 0, default=None)
-            result, './/a[@jsname="UWckNb"]/@href | .//a[contains(@href, "/url?q=")]/@href', 0, default=None
+        if not title or not raw_url:
+            continue
+
+        url = unwrap_google_url(raw_url)
+        thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "SygO9d")]/@src', 0, default="")
+        if "/default.jpg" in thumbnail:
+            thumbnail = thumbnail.split("?")[0].replace("/default.jpg", "/hqdefault.jpg")
+        length = None
+        for span in eval_xpath_list(result, './/span[contains(@class, "YVIcad")]'):
+            length = parse_duration_string(extract_text(span) or "")
+            if length:
+                break
+
+        results.add(
+            results.types.MainResult(
+                url=url,
+                title=title,
+                thumbnail=thumbnail,
+                length=length,
+                iframe_src=get_embeded_stream_url(url) or "",
+                template="videos.html",
             )
-        if url and url.startswith('/url?q='):
-            url = unquote(url[7:].split('&sa=U')[0])
-
-        content = extract_text(
-            eval_xpath_getindex(result, './/div[contains(@class, "ITZIwc")]', 0, default=None), allow_none=True
         )
-        pub_info = extract_text(
-            eval_xpath_getindex(
-                result, './/div[contains(@class, "gqF9jc")] | .//div[contains(@class, "WRu9Cd")]', 0, default=None
-            ),
-            allow_none=True,
-        )
-        # Broader XPath to find any <img> element
-        thumbnail = eval_xpath_getindex(result, './/img/@src', 0, default=None)
-        duration = extract_text(
-            eval_xpath_getindex(result, './/span[contains(@class, "k1U36b")]', 0, default=None), allow_none=True
-        )
-        video_id = eval_xpath_getindex(result, './/div[@jscontroller="rTuANe"]/@data-vid', 0, default=None)
-
-        # Fallback for video_id from URL if not found via XPath
-        if not video_id and url and 'youtube.com' in url:
-            parsed_url = urlparse(url)
-            video_id = parse_qs(parsed_url.query).get('v', [None])[0]
-
-        # Handle thumbnail
-        if thumbnail and thumbnail.startswith('data:image'):
-            img_id = eval_xpath_getindex(result, './/img/@id', 0, default=None)
-            if img_id and img_id in data_image_map:
-                thumbnail = data_image_map[img_id]
-            else:
-                thumbnail = None
-        if not thumbnail and video_id:
-            thumbnail = f"https://img.youtube.com/vi/{video_id}/hqdefault.jpg"
-
-        # Handle video embed URL
-        embed_url = None
-        if video_id:
-            embed_url = get_embeded_stream_url(f"https://www.youtube.com/watch?v={video_id}")
-        elif url:
-            embed_url = get_embeded_stream_url(url)
-
-        # Only append results with valid title and url
-        if title and url:
-            results.append(
-                {
-                    'url': url,
-                    'title': title,
-                    'content': content or '',
-                    'author': pub_info,
-                    'thumbnail': thumbnail,
-                    'length': duration,
-                    'iframe_src': embed_url,
-                    'template': 'videos.html',
-                }
-            )
-
-    # parse suggestion
-    for suggestion in eval_xpath_list(dom, suggestion_xpath):
-        results.append({'suggestion': extract_text(suggestion)})
 
     return results

searx/settings.yml

@@ -1217,12 +1217,12 @@ engines:
   - name: google
     engine: google
     shortcut: go
-    inactive: true
+    disabled: true
 
   - name: google images
     engine: google_images
     shortcut: goi
-    inactive: true
+    disabled: true
 
   - name: google news
     engine: google_news
@@ -1231,7 +1231,6 @@ engines:
   - name: google videos
     engine: google_videos
     shortcut: gov
-    inactive: true
 
   - name: google cse
     engine: google_cse