[feat] results: limit titles and content to fixed length

searx/result_types/_base.py

@@ -36,6 +36,8 @@ from searx.utils import get_embedded_stream_url
 log = logger.getChild("result_types")
 
 WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
+MAXIMUM_TITLE_LENGTH_CHARS = 200
+MAXIMUM_CONTENT_LENGTH_CHARS = 1200
 UNSET = object()
 
 
@@ -92,6 +94,21 @@ def _normalize_url_fields(result: "Result | LegacyResult"):
             ).geturl()
 
 
+def _normalize_text_field(text: str, field_name: str, max_length: int) -> str:
+    if text and not isinstance(text, str):
+        text = str(text)
+        log.debug("result: invalid type of field '%s': %s", field_name, text)
+
+    text = WHITESPACE_REGEX.sub(" ", text).strip()
+
+    # truncate too long result titles and contents
+    if text and len(text) > max_length:
+        # don't split at characters, but at full words instead
+        # e.g. _truncate_text("hello world", 9) yields "hello" instead of "hello wor"
+        text = text[:max_length].rsplit(" ", 1)[0] + " …"
+    return text
+
+
 def _normalize_text_fields(result: "MainResult | LegacyResult"):
 
     # As soon we need LegacyResult not any longer, we can move this function to
@@ -101,18 +118,9 @@ def _normalize_text_fields(result: "MainResult | LegacyResult"):
     # implemented correctly. Historically, however, we have always had a type
     # check here.
 
-    if result.title and not isinstance(result.title, str):
+    result.title = _normalize_text_field(result.title, "title", MAXIMUM_TITLE_LENGTH_CHARS)
-        log.debug("result: invalid type of field 'title': %s", str(result))
+    result.content = _normalize_text_field(result.content, "content", MAXIMUM_CONTENT_LENGTH_CHARS)
-        result.title = str(result)
-    if result.content and not isinstance(result.content, str):
-        log.debug("result: invalid type of field 'content': %s", str(result))
-        result.content = str(result)
 
-    # normalize title and content
-    if result.title:
-        result.title = WHITESPACE_REGEX.sub(" ", result.title).strip()
-    if result.content:
-        result.content = WHITESPACE_REGEX.sub(" ", result.content).strip()
     if result.content == result.title:
         # avoid duplicate content between the content and title fields
         result.content = ""