[fix] add backward compatibility for the "enabled_plugins:"

Before #4183 a builtin plugin was *defautlt_on* when it is listed in the
"enabled_plugins" settings, this patch restores the previous behavior.

Not part of this patch but just to mentioning in context of #4263:

  In the long term, we will abolish the "enabled_plugins:" setting and combine
  all options for the plugins in the "plugins:" setting, as is already planned
  in the PR #4282

Closes: https://github.com/searxng/searxng/issues/4263
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

docs/admin/searx.favicons.rst

@@ -93,15 +93,13 @@ TOML_ configuration is created in the file ``/etc/searxng/favicons.toml``.
 :py:obj:`cache.db_url <.FaviconCacheConfig.db_url>`:
   The path to the (SQLite_) database file.  The default path is in the `/tmp`_
   folder, which is deleted on every reboot and is therefore unsuitable for a
-  production environment.  The FHS_ provides the folder  for the
-  application cache
+  production environment.  The FHS_ provides the folder `/var/cache`_ for the
+  cache of applications, so a suitable storage location of SearXNG's caches is
+  folder ``/var/cache/searxng``.
 
-  The FHS_ provides the folder `/var/cache`_ for the cache of applications, so a
-  suitable storage location of SearXNG's caches is folder ``/var/cache/searxng``.
-  In container systems, a volume should be mounted for this folder and in a
-  standard installation (compare :ref:`create searxng user`), the folder must be
-  created and the user under which the SearXNG process is running must be given
-  write permission to this folder.
+  In a standard installation (compare :ref:`create searxng user`), the folder
+  must be created and the user under which the SearXNG process is running must
+  be given write permission to this folder.
 
   .. code:: bash
 
@@ -109,6 +107,10 @@ TOML_ configuration is created in the file ``/etc/searxng/favicons.toml``.
      $ sudo chown root:searxng /var/cache/searxng/
      $ sudo chmod g+w /var/cache/searxng/
 
+  In container systems, a volume should be mounted for this folder.  Check
+  whether the process in the container has read/write access to the mounted
+  folder.
+
 :py:obj:`cache.LIMIT_TOTAL_BYTES <.FaviconCacheConfig.LIMIT_TOTAL_BYTES>`:
   Maximum of bytes stored in the cache of all blobs.  The limit is only reached
   at each maintenance interval after which the oldest BLOBs are deleted; the

docs/dev/engines/online/soundcloud.rst

@@ -0,0 +1,13 @@
+.. _soundcloud engine:
+
+==========
+Soundcloud
+==========
+
+.. contents::
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.soundcloud
+   :members:

searx/engines/openclipart.py

@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""OpenClipArt (images)"""
+
+from urllib.parse import urlencode
+from lxml import html
+from searx.utils import extract_text, eval_xpath, eval_xpath_list
+
+about = {
+    "website": 'https://openclipart.org/',
+    "wikidata_id": 'Q979593',
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+
+categories = ['images']
+paging = True
+
+base_url = "https://openclipart.org"
+
+
+def request(query, params):
+    args = {
+        'query': query,
+        'p': params['pageno'],
+    }
+    params['url'] = f"{base_url}/search/?{urlencode(args)}"
+    return params
+
+
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    for result in eval_xpath_list(dom, "//div[contains(@class, 'gallery')]/div[contains(@class, 'artwork')]"):
+        results.append(
+            {
+                'template': 'images.html',
+                'url': base_url + extract_text(eval_xpath(result, "./a/@href")),
+                'title': extract_text(eval_xpath(result, "./a/img/@alt")),
+                'img_src': base_url + extract_text(eval_xpath(result, "./a/img/@src")),
+            }
+        )
+
+    return results

searx/engines/soundcloud.py

@@ -1,102 +1,146 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""
- Soundcloud (Music)
-"""
+"""SoundCloud is a German audio streaming service."""
 
 import re
-from json import loads
 from urllib.parse import quote_plus, urlencode
-from lxml import html
+import datetime
+
 from dateutil import parser
+from lxml import html
+
 from searx.network import get as http_get
 
-# about
 about = {
-    "website": 'https://soundcloud.com',
-    "wikidata_id": 'Q568769',
-    "official_api_documentation": 'https://developers.soundcloud.com/',
-    "use_official_api": True,
+    "website": "ttps://soundcloud.com",
+    "wikidata_id": "Q568769",
+    "official_api_documentation": "https://developers.soundcloud.com/docs/api/guide",
+    "use_official_api": False,
     "require_api_key": False,
     "results": 'JSON',
 }
 
-# engine dependent config
-categories = ['music']
+categories = ["music"]
 paging = True
 
-# search-url
-# missing attribute: user_id, app_version, app_locale
-url = 'https://api-v2.soundcloud.com/'
-search_url = (
-    url + 'search?{query}'
-    '&variant_ids='
-    '&facet=model'
-    '&limit=20'
-    '&offset={offset}'
-    '&linked_partitioning=1'
-    '&client_id={client_id}'
-)  # noqa
+search_url = "https://api-v2.soundcloud.com/search"
+"""This is not the offical (developer) url, it is the API which is used by the
+HTML frontend of the common WEB site.
+"""
 
 cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U)
-guest_client_id = ''
+guest_client_id = ""
+results_per_page = 10
+
+soundcloud_facet = "model"
+
+app_locale_map = {
+    "de": "de",
+    "en": "en",
+    "es": "es",
+    "fr": "fr",
+    "oc": "fr",
+    "it": "it",
+    "nl": "nl",
+    "pl": "pl",
+    "szl": "pl",
+    "pt": "pt_BR",
+    "pap": "pt_BR",
+    "sv": "sv",
+}
 
 
-def get_client_id():
-    resp = http_get("https://soundcloud.com")
-
-    if resp.ok:
-        tree = html.fromstring(resp.content)
-        # script_tags has been moved from /assets/app/ to /assets/ path.  I
-        # found client_id in https://a-v2.sndcdn.com/assets/49-a0c01933-3.js
-        script_tags = tree.xpath("//script[contains(@src, '/assets/')]")
-        app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]
-
-        # extracts valid app_js urls from soundcloud.com content
-        for app_js_url in app_js_urls[::-1]:
-            # gets app_js and searches for the clientid
-            resp = http_get(app_js_url)
-            if resp.ok:
-                cids = cid_re.search(resp.content.decode())
-                if cids is not None and len(cids.groups()):
-                    return cids.groups()[0]
-    logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")
-    return ""
-
-
-def init(engine_settings=None):  # pylint: disable=unused-argument
-    global guest_client_id  # pylint: disable=global-statement
-    # api-key
-    guest_client_id = get_client_id()
-
-
-# do search-request
 def request(query, params):
-    offset = (params['pageno'] - 1) * 20
 
-    params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset, client_id=guest_client_id)
+    # missing attributes: user_id, app_version
+    # - user_id=451561-497874-703312-310156
+    # - app_version=1740727428
 
+    args = {
+        "q": query,
+        "offset": (params['pageno'] - 1) * results_per_page,
+        "limit": results_per_page,
+        "facet": soundcloud_facet,
+        "client_id": guest_client_id,
+        "app_locale": app_locale_map.get(params["language"].split("-")[0], "en"),
+    }
+
+    params['url'] = f"{search_url}?{urlencode(args)}"
     return params
 
 
 def response(resp):
     results = []
-    search_res = loads(resp.text)
+    data = resp.json()
 
-    # parse results
-    for result in search_res.get('collection', []):
+    for result in data.get("collection", []):
 
-        if result['kind'] in ('track', 'playlist'):
-            uri = quote_plus(result['uri'])
+        if result["kind"] in ("track", "playlist"):
+            url = result.get("permalink_url")
+            if not url:
+                continue
+            uri = quote_plus(result.get("uri"))
+            content = [
+                result.get("description"),
+                result.get("label_name"),
+            ]
             res = {
-                'url': result['permalink_url'],
-                'title': result['title'],
-                'content': result['description'] or '',
-                'publishedDate': parser.parse(result['last_modified']),
-                'iframe_src': "https://w.soundcloud.com/player/?url=" + uri,
+                "url": url,
+                "title": result["title"],
+                "content": " / ".join([c for c in content if c]),
+                "publishedDate": parser.parse(result["last_modified"]),
+                "iframe_src": "https://w.soundcloud.com/player/?url=" + uri,
+                "views": result.get("likes_count"),
             }
-            thumbnail = result['artwork_url'] or result['user']['avatar_url']
-            if thumbnail:
-                res['thumbnail'] = thumbnail
+            thumbnail = result["artwork_url"] or result["user"]["avatar_url"]
+            res["thumbnail"] = thumbnail or None
+            length = int(result.get("duration", 0) / 1000)
+            if length:
+                length = datetime.timedelta(seconds=length)
+                res["length"] = length
+            res["views"] = result.get("playback_count", 0) or None
+            res["author"] = result.get("user", {}).get("full_name") or None
             results.append(res)
 
     return results
+
+
+def init(engine_settings=None):  # pylint: disable=unused-argument
+    global guest_client_id  # pylint: disable=global-statement
+    guest_client_id = get_client_id()
+
+
+def get_client_id() -> str:
+
+    client_id = ""
+    url = "https://soundcloud.com"
+    resp = http_get(url, timeout=10)
+
+    if not resp.ok:
+        logger.error("init: GET %s failed", url)
+        return client_id
+
+    tree = html.fromstring(resp.content)
+    script_tags = tree.xpath("//script[contains(@src, '/assets/')]")
+    app_js_urls = [tag.get("src") for tag in script_tags if tag is not None]
+
+    # extracts valid app_js urls from soundcloud.com content
+
+    for url in app_js_urls[::-1]:
+
+        # gets app_js and search for the client_id
+        resp = http_get(url)
+
+        if not resp.ok:
+            logger.error("init: app_js GET %s failed", url)
+            continue
+
+        cids = cid_re.search(resp.content.decode())
+        if cids and len(cids.groups()):
+            client_id = cids.groups()[0]
+            break
+
+    if client_id:
+        logger.info("using client_id '%s' for soundclud queries", client_id)
+    else:
+        logger.warning("missing valid client_id for soundclud queries")
+    return client_id

searx/plugins/_core.py

@@ -72,10 +72,10 @@ class PluginInfo:
 class Plugin(abc.ABC):
     """Abstract base class of all Plugins."""
 
-    id: typing.ClassVar[str]
+    id: str = ""
     """The ID (suffix) in the HTML form."""
 
-    default_on: typing.ClassVar[bool]
+    default_on: bool = False
     """Plugin is enabled/disabled by default."""
 
     keywords: list[str] = []
@@ -91,8 +91,12 @@ class Plugin(abc.ABC):
     info: PluginInfo
     """Informations about the *plugin*, see :py:obj:`PluginInfo`."""
 
+    fqn: str = ""
+
     def __init__(self) -> None:
         super().__init__()
+        if not self.fqn:
+            self.fqn = self.__class__.__mro__[0].__module__
 
         for attr in ["id", "default_on"]:
             if getattr(self, attr, None) is None:
@@ -178,10 +182,11 @@ class ModulePlugin(Plugin):
 
     _required_attrs = (("name", str), ("description", str), ("default_on", bool))
 
-    def __init__(self, mod: types.ModuleType):
+    def __init__(self, mod: types.ModuleType, fqn: str):
         """In case of missing attributes in the module or wrong types are given,
         a :py:obj:`TypeError` exception is raised."""
 
+        self.fqn = fqn
         self.module = mod
         self.id = getattr(self.module, "plugin_id", self.module.__name__)
         self.log = logging.getLogger(self.module.__name__)
@@ -282,7 +287,7 @@ class PluginStorage:
 
             # for backward compatibility
             mod = load_module(f.name, str(f.parent))
-            self.register(ModulePlugin(mod))
+            self.register(ModulePlugin(mod, f"searx.plugins.{f.stem}"))
 
         for fqn in searx.get_setting("plugins"):  # type: ignore
             self.register_by_fqn(fqn)
@@ -297,8 +302,35 @@ class PluginStorage:
             plugin.log.critical(msg)
             raise KeyError(msg)
 
+        if not plugin.fqn.startswith("searx.plugins."):
+            self.plugin_list.add(plugin)
+            plugin.log.debug("plugin has been registered")
+            return
+
+        # backward compatibility for the enabled_plugins setting
+        # https://docs.searxng.org/admin/settings/settings_plugins.html#enabled-plugins-internal
+        en_plgs: list[str] | None = searx.get_setting("enabled_plugins")  # type:ignore
+
+        if en_plgs is None:
+            # enabled_plugins not listed in the /etc/searxng/settings.yml:
+            # check default_on before register ..
+            if plugin.default_on:
+                self.plugin_list.add(plugin)
+                plugin.log.debug("builtin plugin has been registered by SearXNG's defaults")
+                return
+            plugin.log.debug("builtin plugin is not registered by SearXNG's defaults")
+            return
+
+        if plugin.info.name not in en_plgs:
+            # enabled_plugins listed in the /etc/searxng/settings.yml,
+            # but this plugin is not listed in:
+            plugin.log.debug("builtin plugin is not registered by maintainer's settings")
+            return
+
+        # if the plugin is in enabled_plugins, then it is on by default.
+        plugin.default_on = True
         self.plugin_list.add(plugin)
-        plugin.log.debug("plugin has been loaded")
+        plugin.log.debug("builtin plugin is registered by maintainer's settings")
 
     def register_by_fqn(self, fqn: str):
         """Register a :py:obj:`Plugin` via its fully qualified class name (FQN).
@@ -324,7 +356,8 @@ class PluginStorage:
             warnings.warn(
                 f"plugin {fqn} is implemented in a legacy module / migrate to searx.plugins.Plugin", DeprecationWarning
             )
-            self.register(ModulePlugin(code_obj))
+
+            self.register(ModulePlugin(code_obj, fqn))
             return
 
         self.register(code_obj())

searx/settings.yml

@@ -1361,6 +1361,13 @@ engines:
       require_api_key: false
       results: JSON
 
+  - name: openclipart
+    engine: openclipart
+    shortcut: ocl
+    inactive: true
+    disabled: true
+    timeout: 30
+
   - name: openlibrary
     engine: openlibrary
     shortcut: ol

tests/unit/test_plugin_calculator.py

@@ -26,7 +26,7 @@ class PluginCalculator(SearxTestCase):
         engines = {}
 
         self.storage = searx.plugins.PluginStorage()
-        self.storage.register(ModulePlugin(mod))
+        self.storage.register(ModulePlugin(mod, "searx.plugins.calculator"))
         self.storage.init(self.app)
         self.pref = searx.preferences.Preferences(["simple"], ["general"], engines, self.storage)
         self.pref.parse_dict({"locale": "en"})