[fix] presearch engine: domain sometimes included in beginning of titles

As long we don't have a working solution for cache busting [3][4] we should not
set an expire time in thw uWSGI config.

The default procedure in every web browser is the "304 Not Modified" [2] and
this default procedure should also be sufficient for us as long as we have not
implemented a complete alternative (cache busting) / form [1]

> By default uWSGI will add a Last-Modified [2] header to all static responses,
> and will honor the If-Modified-Since [2] request header.

[1] https://uwsgi-docs.readthedocs.io/en/latest/StaticFiles.html#setting-the-expires-headers
[2] https://developer.mozilla.org/de/docs/Web/HTTP/Status/304
[3] https://github.com/searxng/searxng/pull/4433
[4] https://github.com/searxng/searxng/issues/964

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

dockerfiles/uwsgi.ini

@@ -49,6 +49,5 @@ die-on-term
 # uwsgi serves the static files
 static-map = /static=/usr/local/searxng/searx/static
 # expires set to one day
-static-expires = /* 86400
 static-gzip-all = True
 offload-threads = %k

searx/engines/360search_videos.py

@@ -6,7 +6,7 @@ from urllib.parse import urlencode
 from datetime import datetime
 
 from searx.exceptions import SearxEngineAPIException
-from searx.utils import html_to_text
+from searx.utils import html_to_text, get_embeded_stream_url
 
 about = {
     "website": "https://tv.360kan.com/",
@@ -58,6 +58,7 @@ def response(resp):
                 'template': 'videos.html',
                 'publishedDate': published_date,
                 'thumbnail': entry["cover_img"],
+                "iframe_src": get_embeded_stream_url(entry["play_url"]),
             }
         )
 

searx/engines/presearch.py

@@ -64,7 +64,7 @@ Implementations
 
 """
 
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urlparse
 from searx import locales
 from searx.network import get
 from searx.utils import gen_useragent, html_to_text
@@ -155,13 +155,34 @@ def _strip_leading_strings(text):
     return text.strip()
 
 
+def _fix_title(title, url):
+    """
+    Titles from Presearch shows domain + title without spacing, and HTML
+    This function removes these 2 issues.
+    Transforming "translate.google.co.in<em>Google</em> Translate" into "Google Translate"
+    """
+    parsed_url = urlparse(url)
+    domain = parsed_url.netloc
+    title = html_to_text(title)
+    # Fixes issue where domain would show up in the title
+    # translate.google.co.inGoogle Translate -> Google Translate
+    if (
+        title.startswith(domain)
+        and len(title) > len(domain)
+        and not title.startswith(domain + "/")
+        and not title.startswith(domain + " ")
+    ):
+        title = title.removeprefix(domain)
+    return title
+
+
 def parse_search_query(json_results):
     results = []
 
     for item in json_results.get('specialSections', {}).get('topStoriesCompact', {}).get('data', []):
         result = {
             'url': item['link'],
-            'title': html_to_text(item['title']),
+            'title': _fix_title(item['title'], item['link']),
             'thumbnail': item['image'],
             'content': '',
             'metadata': item.get('source'),
@@ -171,7 +192,7 @@ def parse_search_query(json_results):
     for item in json_results.get('standardResults', []):
         result = {
             'url': item['link'],
-            'title': html_to_text(item['title']),
+            'title': _fix_title(item['title'], item['link']),
             'content': html_to_text(item['description']),
         }
         results.append(result)

searx/utils.py

@@ -633,7 +633,7 @@ def _get_fasttext_model() -> "fasttext.FastText._FastText":  # type: ignore
 def get_embeded_stream_url(url):
     """
     Converts a standard video URL into its embed format. Supported services include Youtube,
-    Facebook, Instagram, TikTok, and Dailymotion.
+    Facebook, Instagram, TikTok, Dailymotion, and Bilibili.
     """
     parsed_url = urlparse(url)
     iframe_src = None
@@ -673,6 +673,22 @@ def get_embeded_stream_url(url):
             video_id = path_parts[2]
             iframe_src = 'https://www.dailymotion.com/embed/video/' + video_id
 
+    # Bilibili
+    elif parsed_url.netloc in ['www.bilibili.com', 'bilibili.com'] and parsed_url.path.startswith('/video/'):
+        path_parts = parsed_url.path.split('/')
+
+        video_id = path_parts[2]
+        param_key = None
+        if video_id.startswith('av'):
+            video_id = video_id[2:]
+            param_key = 'aid'
+        elif video_id.startswith('BV'):
+            param_key = 'bvid'
+
+        iframe_src = (
+            f'https://player.bilibili.com/player.html?{param_key}={video_id}&high_quality=1&autoplay=false&danmaku=0'
+        )
+
     return iframe_src
 
 

utils/templates/etc/uwsgi/apps-archlinux/searxng.ini

@@ -81,7 +81,5 @@ buffer-size = 8192
 #     static_use_hash: true
 #
 static-map = /static=${SEARXNG_STATIC}
-# expires set to one day
-static-expires = /* 86400
 static-gzip-all = True
 offload-threads = %k

utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket

@@ -78,7 +78,5 @@ buffer-size = 8192
 #     static_use_hash: true
 #
 static-map = /static=${SEARXNG_STATIC}
-# expires set to one day
-static-expires = /* 86400
 static-gzip-all = True
 offload-threads = %k

utils/templates/etc/uwsgi/apps-available/searxng.ini

@@ -84,7 +84,5 @@ buffer-size = 8192
 #     static_use_hash: true
 #
 static-map = /static=${SEARXNG_STATIC}
-# expires set to one day
-static-expires = /* 86400
 static-gzip-all = True
 offload-threads = %k

utils/templates/etc/uwsgi/apps-available/searxng.ini:socket

@@ -81,7 +81,5 @@ buffer-size = 8192
 #     static_use_hash: true
 #
 static-map = /static=${SEARXNG_STATIC}
-# expires set to one day
-static-expires = /* 86400
 static-gzip-all = True
 offload-threads = %k