[chore] Google (HTML) engine: remove obsolete GSA User-Agents (#6366)

The GSA headers that were introduced in PR #5644 unfortunately no longer
work (#6359).

The Google engines

- google.py
- google_videos.py

do not work anymore either, but we'll leave it in the code for now:

- In google.py, central functions like get_google_info(..) are provided, which
  are also used by other modules.
- We will probably need a Google HTML (and video) engine again very soon.

Related:

- https://github.com/searxng/searxng/issues/6359
- https://github.com/searxng/searxng/pull/6364

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

.github/workflows/data-update.yml

@@ -33,7 +33,6 @@ jobs:
           - update_engine_traits.py
           - update_wikidata_units.py
           - update_engine_descriptions.py
-          - update_gsa_useragents.py
 
     permissions:
       contents: write

Makefile

@@ -63,7 +63,7 @@ format: format.python format.shell
 # wrap ./manage script
 
 MANAGE += weblate.translations.commit weblate.push.translations
-MANAGE += data.all data.traits data.useragents data.gsa_useragents data.locales data.currencies
+MANAGE += data.all data.traits data.useragents data.locales data.currencies
 MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
 MANAGE += podman.build
 MANAGE += docker.build docker.buildx

searx/data/__init__.py

@@ -6,7 +6,7 @@ make data.all
 """
 # pylint: disable=invalid-name
 
-__all__ = ["ahmia_blacklist_loader", "gsa_useragents_loader", "data_dir", "get_cache"]
+__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
 
 import json
 import typing as t
@@ -63,7 +63,6 @@ lazy_globals = {
     "ENGINE_TRAITS": None,
     "LOCALES": None,
     "TRACKER_PATTERNS": TrackerPatternsDB(),
-    "GSA_USER_AGENTS": None,
 }
 
 data_json_files = {
@@ -106,24 +105,3 @@ def ahmia_blacklist_loader() -> list[str]:
     """
     with open(data_dir / 'ahmia_blacklist.txt', encoding='utf-8') as f:
         return f.read().split()
-
-
-def gsa_useragents_loader() -> list[str]:
-    """Load data from `gsa_useragents.txt` and return a list of user agents
-    suitable for Google.  The user agents are fetched by::
-
-      searxng_extra/update/update_gsa_useragents.py
-
-    This function is used by :py:mod:`searx.engines.google`.
-
-    """
-    data = lazy_globals["GSA_USER_AGENTS"]
-    if data is not None:
-        return data
-
-    log.debug("init searx.data.%s", "GSA_USER_AGENTS")
-
-    with open(data_dir / 'gsa_useragents.txt', encoding='utf-8') as f:
-        lazy_globals["GSA_USER_AGENTS"] = f.read().splitlines()
-
-    return lazy_globals["GSA_USER_AGENTS"]

searx/engines/google.py

@@ -32,7 +32,6 @@ from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
     extract_text,
-    # gen_gsa_useragent,
 )
 
 if t.TYPE_CHECKING:
@@ -268,11 +267,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     # HTTP headers
 
     ret_val["headers"]["Accept"] = "*/*"
-    # GSA for iPhone useragent had been added in
-    # - https://github.com/searxng/searxng/pull/5644
-    # but doe no longer work
-    # - https://github.com/searxng/searxng/issues/6359
-    # ret_val["headers"]["User-Agent"] = gen_gsa_useragent()
 
     # Cookies
 

searx/settings.yml

@@ -1216,6 +1216,7 @@ engines:
   - name: google videos
     engine: google_videos
     shortcut: gov
+    inactive: true
 
   - name: google scholar
     engine: google_scholar

searx/utils.py

@@ -26,7 +26,7 @@ from lxml.etree import XPath, XPathError, XPathSyntaxError
 from lxml.etree import ElementBase, _Element  # pyright: ignore[reportPrivateUsage]
 
 from searx import settings
-from searx.data import USER_AGENTS, gsa_useragents_loader
+from searx.data import USER_AGENTS
 from searx.version import VERSION_TAG
 from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
 from searx import logger
@@ -82,14 +82,6 @@ def gen_useragent(os_string: str | None = None) -> str:
     )
 
 
-def gen_gsa_useragent() -> str:
-    """Return a random "Google Go App" User Agent suitable for Google
-
-    See searx/data/gsa_useragents.txt
-    """
-    return choice(gsa_useragents_loader()) + " NSTNWV"
-
-
 class HTMLTextExtractor(HTMLParser):
     """Internal class to extract text from HTML"""
 

searxng_extra/update/update_gsa_useragents.py

@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""This script fetches user agents suitable for Google.
-
-Output file: :origin:`searx/data/gsa_useragents.txt` (:origin:`CI Update data
-...  <.github/workflows/data-update.yml>`).
-
-.. Source for user agents: https://github.com/intoli/user-agents/
-
-"""
-
-from gzip import decompress
-from json import loads
-
-from searx.data import data_dir
-from searx.network import get as http_get
-from searx.utils import searxng_useragent
-
-DATA_FILE = data_dir / "gsa_useragents.txt"
-URL = "https://raw.githubusercontent.com/intoli/user-agents/main/src/user-agents.json.gz"
-
-
-def fetch_gsa_useragents() -> list[str]:
-    response = http_get(URL, timeout=3.0, headers={"User-Agent": searxng_useragent()})
-    response.raise_for_status()
-
-    suas: set[str] = set()
-    for ua in loads(decompress(response.content)):
-        if (
-            "Android" in ua["userAgent"]
-            and "Chrome" in ua["userAgent"]
-            and "Samsung" not in ua["userAgent"]
-            and "Android 10; K" not in ua["userAgent"]
-        ):
-            suas.add(ua["userAgent"])
-
-    luas = list(suas)
-    luas.sort()
-
-    return luas
-
-
-if __name__ == "__main__":
-    useragents = fetch_gsa_useragents()
-    with DATA_FILE.open("w", encoding="utf-8") as f:
-        f.write("\n".join(useragents))

utils/lib_sxng_data.sh

@@ -7,7 +7,6 @@ data.:
   all           : update searx/sxng_locales.py and searx/data/*
   traits        : update searx/data/engine_traits.json & searx/sxng_locales.py
   useragents    : update searx/data/useragents.json with the most recent versions of Firefox
-  gsa_useragents: update searx/data/gsa_useragents.txt with compatible useragents
   locales       : update searx/data/locales.json from babel
   currencies    : update searx/data/currencies.json from wikidata
 EOF
@@ -20,7 +19,6 @@ data.all() {
         pyenv.activate
         data.traits
         data.useragents
-        data.gsa_useragents
         data.locales
 
         build_msg DATA "update searx/data/osm_keys_tags.json"
@@ -55,12 +53,6 @@ data.useragents() {
     dump_return $?
 }
 
-data.gsa_useragents() {
-    build_msg DATA "update searx/data/gsa_useragents.txt"
-    pyenv.cmd python searxng_extra/update/update_gsa_useragents.py
-    dump_return $?
-}
-
 data.locales() {
     (
         set -e