[chore] Google (HTML) engine: remove obsolete GSA User-Agents (#6366)

The GSA headers that were introduced in PR #5644 unfortunately no longer
work (#6359).

The Google engines

- google.py
- google_videos.py

do not work anymore either, but we'll leave it in the code for now:

- In google.py, central functions like get_google_info(..) are provided, which
  are also used by other modules.
- We will probably need a Google HTML (and video) engine again very soon.

Related:

- https://github.com/searxng/searxng/issues/6359
- https://github.com/searxng/searxng/pull/6364

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/data/__init__.py

@@ -6,7 +6,7 @@ make data.all
 """
 # pylint: disable=invalid-name
 
-__all__ = ["ahmia_blacklist_loader", "gsa_useragents_loader", "data_dir", "get_cache"]
+__all__ = ["ahmia_blacklist_loader", "data_dir", "get_cache"]
 
 import json
 import typing as t
@@ -63,7 +63,6 @@ lazy_globals = {
     "ENGINE_TRAITS": None,
     "LOCALES": None,
     "TRACKER_PATTERNS": TrackerPatternsDB(),
-    "GSA_USER_AGENTS": None,
 }
 
 data_json_files = {
@@ -106,24 +105,3 @@ def ahmia_blacklist_loader() -> list[str]:
     """
     with open(data_dir / 'ahmia_blacklist.txt', encoding='utf-8') as f:
         return f.read().split()
-
-
-def gsa_useragents_loader() -> list[str]:
-    """Load data from `gsa_useragents.txt` and return a list of user agents
-    suitable for Google.  The user agents are fetched by::
-
-      searxng_extra/update/update_gsa_useragents.py
-
-    This function is used by :py:mod:`searx.engines.google`.
-
-    """
-    data = lazy_globals["GSA_USER_AGENTS"]
-    if data is not None:
-        return data
-
-    log.debug("init searx.data.%s", "GSA_USER_AGENTS")
-
-    with open(data_dir / 'gsa_useragents.txt', encoding='utf-8') as f:
-        lazy_globals["GSA_USER_AGENTS"] = f.read().splitlines()
-
-    return lazy_globals["GSA_USER_AGENTS"]

searx/engines/google.py

@@ -32,7 +32,6 @@ from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
     extract_text,
-    # gen_gsa_useragent,
 )
 
 if t.TYPE_CHECKING:
@@ -268,11 +267,6 @@ def get_google_info(params: "OnlineParams", eng_traits: EngineTraits) -> dict[st
     # HTTP headers
 
     ret_val["headers"]["Accept"] = "*/*"
-    # GSA for iPhone useragent had been added in
-    # - https://github.com/searxng/searxng/pull/5644
-    # but doe no longer work
-    # - https://github.com/searxng/searxng/issues/6359
-    # ret_val["headers"]["User-Agent"] = gen_gsa_useragent()
 
     # Cookies
 

searx/settings.yml

@@ -1216,6 +1216,7 @@ engines:
   - name: google videos
     engine: google_videos
     shortcut: gov
+    inactive: true
 
   - name: google scholar
     engine: google_scholar

searx/utils.py

@@ -26,7 +26,7 @@ from lxml.etree import XPath, XPathError, XPathSyntaxError
 from lxml.etree import ElementBase, _Element  # pyright: ignore[reportPrivateUsage]
 
 from searx import settings
-from searx.data import USER_AGENTS, gsa_useragents_loader
+from searx.data import USER_AGENTS
 from searx.version import VERSION_TAG
 from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
 from searx import logger
@@ -82,14 +82,6 @@ def gen_useragent(os_string: str | None = None) -> str:
     )
 
 
-def gen_gsa_useragent() -> str:
-    """Return a random "Google Go App" User Agent suitable for Google
-
-    See searx/data/gsa_useragents.txt
-    """
-    return choice(gsa_useragents_loader()) + " NSTNWV"
-
-
 class HTMLTextExtractor(HTMLParser):
     """Internal class to extract text from HTML"""