[mod] engines: migrate video engines away from get_embedded_stream_url

Currently, many engines that provide video results call the same
`get_embedded_stream_url` method for setting the `iframe_src`. There
is no value in that because this logic is engine-specific and probably
many video engines forgot to implement this.

With these changes, it's done automatically, so engine implementors
don't have to worry about setting an `iframe_src` (unless the engine
explicitly has a field for it).

searx/engines/360search_videos.py

@@ -6,7 +6,8 @@ from urllib.parse import urlencode
 from datetime import datetime
 
 from searx.exceptions import SearxEngineAPIException
-from searx.utils import html_to_text, get_embeded_stream_url
+from searx.result_types import EngineResults
+from searx.utils import html_to_text
 
 about = {
     "website": "https://tv.360kan.com/",
@@ -29,12 +30,12 @@ def request(query, params):
     return params
 
 
-def response(resp):
+def response(resp) -> EngineResults:
     try:
         data = resp.json()
     except Exception as e:
         raise SearxEngineAPIException(f"Invalid response: {e}") from e
-    results = []
+    res = EngineResults()
 
     if "data" not in data or "result" not in data["data"]:
         raise SearxEngineAPIException("Invalid response")
@@ -50,16 +51,15 @@ def response(resp):
             except (ValueError, TypeError):
                 published_date = None
 
-        results.append(
-            {
-                'url': entry["play_url"],
-                'title': html_to_text(entry["title"]),
-                'content': html_to_text(entry["description"]),
-                'template': 'videos.html',
-                'publishedDate': published_date,
-                'thumbnail': entry["cover_img"],
-                "iframe_src": get_embeded_stream_url(entry["play_url"]),
-            }
+        res.add(
+            res.types.LegacyResult(
+                url=entry["play_url"],
+                title=html_to_text(entry["title"]),
+                content=html_to_text(entry["description"]),
+                template='videos.html',
+                publishedDate=published_date,
+                thumbnail=entry["cover_img"],
+            )
         )
 
-    return results
+    return res

searx/engines/brave.py

@@ -135,7 +135,7 @@ from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
     extract_text,
-    get_embeded_stream_url,
+    get_embedded_stream_url,
     js_obj_str_to_json_str,
     js_obj_str_to_python,
 )
@@ -338,7 +338,7 @@ def _parse_search(resp: SXNG_Response) -> EngineResults:
         if len(video_tag):
             # In my tests a video tag in the WEB search was most often not a
             # video, except the ones from youtube ..
-            iframe_src = get_embeded_stream_url(url)
+            iframe_src = get_embedded_stream_url(url)
             if iframe_src:
                 item["iframe_src"] = iframe_src
                 item["template"] = "videos.html"
@@ -406,9 +406,6 @@ def _parse_videos(json_resp: dict[str, t.Any]) -> EngineResults:
         )
         if result["thumbnail"] is not None:
             item["thumbnail"] = result["thumbnail"]["src"]
-        iframe_src = get_embeded_stream_url(result["url"])
-        if iframe_src:
-            item["iframe_src"] = iframe_src
 
         res.add(item)
 

searx/engines/duckduckgo_extra.py

@@ -10,7 +10,8 @@ from datetime import datetime
 from urllib.parse import urlencode
 from urllib.parse import quote_plus
 
-from searx.utils import get_embeded_stream_url, html_to_text, gen_useragent, extr
+from searx.result_types import EngineResults, MainResult, LegacyResult, Image
+from searx.utils import html_to_text, gen_useragent, extr
 from searx.network import get  # see https://github.com/searxng/searxng/issues/762
 
 from searx.engines.duckduckgo import fetch_traits  # pylint: disable=unused-import
@@ -148,54 +149,51 @@ def request(query: str, params: "OnlineParams") -> None:
 
 
 def _image_result(result):
-    return {
-        'template': 'images.html',
-        'url': result['url'],
-        'title': result['title'],
-        'content': '',
-        'thumbnail_src': result['thumbnail'],
-        'img_src': result['image'],
-        'resolution': '%s x %s' % (result['width'], result['height']),
-        'source': result['source'],
-    }
+    return Image(
+        url=result['url'],
+        title=result['title'],
+        content='',
+        thumbnail_src=result['thumbnail'],
+        img_src=result['image'],
+        resolution='%s x %s' % (result['width'], result['height']),
+        source=result['source'],
+    )
 
 
 def _video_result(result):
-    return {
-        'template': 'videos.html',
-        'url': result['content'],
-        'title': result['title'],
-        'content': result['description'],
-        'thumbnail': result['images'].get('small') or result['images'].get('medium'),
-        'iframe_src': get_embeded_stream_url(result['content']),
-        'source': result['provider'],
-        'length': result['duration'],
-        'metadata': result.get('uploader'),
-    }
+    return LegacyResult(
+        template='videos.html',
+        url=result['content'],
+        title=result['title'],
+        content=result['description'],
+        thumbnail=result['images'].get('small') or result['images'].get('medium'),
+        source=result['provider'],
+        length=result['duration'],
+        metadata=result.get('uploader'),
+    )
 
 
 def _news_result(result):
-    return {
-        'url': result['url'],
-        'title': result['title'],
-        'content': html_to_text(result['excerpt']),
-        'source': result['source'],
-        'publishedDate': datetime.fromtimestamp(result['date']),
-    }
+    return MainResult(
+        url=result['url'],
+        title=result['title'],
+        content=html_to_text(result['excerpt']),
+        publishedDate=datetime.fromtimestamp(result['date']),
+    )
 
 
-def response(resp):
-    results = []
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
     res_json = resp.json()
 
     for result in res_json['results']:
         if ddg_category == 'images':
-            results.append(_image_result(result))
+            res.add(_image_result(result))
         elif ddg_category == 'videos':
-            results.append(_video_result(result))
+            res.add(_video_result(result))
         elif ddg_category == 'news':
-            results.append(_news_result(result))
+            res.add(_news_result(result))
         else:
             raise ValueError(f"Invalid duckduckgo category: {ddg_category}")
 
-    return results
+    return res

searx/engines/google_videos.py

@@ -10,7 +10,6 @@ from searx.utils import (
     eval_xpath_getindex,
     eval_xpath_list,
     extract_text,
-    get_embeded_stream_url,
     parse_duration_string,
 )
 
@@ -79,7 +78,6 @@ def response(resp: "SXNG_Response") -> EngineResults:
                 title=title,
                 thumbnail=thumbnail,
                 length=length,
-                iframe_src=get_embeded_stream_url(url) or "",
                 template="videos.html",
             )
         )

searx/engines/naver.py

@@ -18,7 +18,6 @@ from searx.utils import (
     html_to_text,
     parse_duration_string,
     js_obj_str_to_python,
-    get_embeded_stream_url,
 )
 
 # engine metadata
@@ -195,7 +194,7 @@ def parse_news(data):
 
 
 def parse_videos(data):
-    results = []
+    res = EngineResults()
 
     dom = html.fromstring(data)
 
@@ -214,15 +213,14 @@ def parse_videos(data):
         except (ValueError, TypeError):
             pass
 
-        results.append(
-            {
-                "template": "videos.html",
-                "title": extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
-                "url": url,
-                "thumbnail": thumbnail,
-                "length": length,
-                "iframe_src": get_embeded_stream_url(url),
-            }
+        res.add(
+            res.types.LegacyResult(
+                template="videos.html",
+                title=extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
+                url=url,
+                thumbnail=thumbnail,
+                length=length,
+            )
         )
 
-    return results
+    return res

searx/engines/privacywall.py

@@ -14,7 +14,7 @@ from lxml import html
 import babel
 
 from searx.enginelib.traits import EngineTraits
-from searx.utils import eval_xpath_list, eval_xpath, extract_text, get_embeded_stream_url, extr
+from searx.utils import eval_xpath_list, eval_xpath, extract_text, extr
 from searx.locales import region_tag
 from searx.result_types import EngineResults
 
@@ -154,7 +154,6 @@ def _video_results(doc: "ElementBase") -> EngineResults:
                 title=extract_text(eval_xpath(result, ".//h2[contains(@class, 'video-card-title')]")) or "",
                 content=extract_text(eval_xpath(result, ".//p")) or "",
                 thumbnail=thumbnail or "",
-                iframe_src=get_embeded_stream_url(url) or "",
             )
         )
 

searx/engines/qwant.py

@@ -60,9 +60,6 @@ from searx.exceptions import (
     SearxEngineTooManyRequestsException,
 )
 from searx.network import raise_for_httperror
-from searx.utils import (
-    get_embeded_stream_url,
-)
 from searx.result_types import EngineResults
 
 if t.TYPE_CHECKING:
@@ -299,7 +296,6 @@ def response(resp: "SXNG_Response") -> EngineResults:
                         title=title,
                         url=res_url,
                         content=content,
-                        iframe_src=get_embeded_stream_url(res_url),
                         publishedDate=pub_date,
                         thumbnail=thumbnail,
                         template="videos.html",

searx/engines/tonline.py

@@ -14,7 +14,7 @@ from urllib.parse import urlencode
 
 from lxml import html
 
-from searx.utils import eval_xpath_list, eval_xpath, extract_text, get_embeded_stream_url, ElementType
+from searx.utils import eval_xpath_list, eval_xpath, extract_text, ElementType
 from searx.result_types import EngineResults
 from searx.enginelib import EngineAbout
 
@@ -126,7 +126,6 @@ def _video_results(doc: ElementType, res: EngineResults):
                 url=url,
                 title=" - ".join(extract_text(part) or "" for part in title_parts),
                 thumbnail=extract_text(eval_xpath(result, ".//img/@src") or "") or "",
-                iframe_src=get_embeded_stream_url(url) or "",
             )
         )
 

searx/result_types/__init__.py

@@ -15,6 +15,7 @@
 __all__ = [
     "Result",
     "MainResult",
+    "LegacyResult",
     "KeyValue",
     "EngineResults",
     "AnswerSet",

searx/result_types/_base.py

@@ -31,6 +31,7 @@ from collections.abc import Callable
 import msgspec
 
 from searx import logger
+from searx.utils import get_embedded_stream_url
 
 log = logger.getChild("result_types")
 
@@ -480,6 +481,7 @@ class LegacyResult(dict[str, t.Any]):
     category: str
     publishedDate: datetime.datetime | None
     pubdate: str = ""
+    iframe_src: str | None
 
     # infobox result
     urls: list[dict[str, str]]
@@ -509,6 +511,7 @@ class LegacyResult(dict[str, t.Any]):
         self["score"] = self.get("score", 0)
         self["category"] = self.get("category", "")
         self["publishedDate"] = self.get("publishedDate")
+        self["iframe_src"] = self.get("iframe_src")
 
         if "infobox" in self:
             self["urls"] = self.get("urls", [])
@@ -531,6 +534,10 @@ class LegacyResult(dict[str, t.Any]):
                 DeprecationWarning,
             )
 
+        # TODO: move into typed video results class once it is implemented  # pylint: disable=fixme
+        if self.template == "videos.html" and self.url and not self.iframe_src:
+            self.iframe_src = get_embedded_stream_url(self.url)
+
     def __getattr__(self, name: str, default: t.Any = UNSET) -> t.Any:
         if default == UNSET and name not in self:
             raise AttributeError(f"LegacyResult object has no field named: {name}")

searx/utils.py

@@ -592,7 +592,7 @@ def eval_xpath_getindex(
     return default
 
 
-def get_embeded_stream_url(url: str):
+def get_embedded_stream_url(url: str):
     """
     Converts a standard video URL into its embed format. Supported services include Youtube,
     Facebook, Instagram, TikTok, Dailymotion, and Bilibili.