jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2026-09-13 17:56:05 +00:00
Code Issues Projects Releases Wiki Activity

[feat] results: limit titles and content to fixed length

Browse Source
This commit is contained in:
Bnyro
2026-09-13 12:30:45 +02:00
parent 3b80090654
commit 42829cecd6
1 changed files with 19 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
searx/result_types/_base.py
View File

@@ -36,6 +36,8 @@ from searx.utils import get_embedded_stream_url
log = logger.getChild("result_types")
WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
MAXIMUM_TITLE_LENGTH_CHARS = 200
MAXIMUM_CONTENT_LENGTH_CHARS = 1200
UNSET = object()
@@ -92,6 +94,21 @@ def _normalize_url_fields(result: "Result | LegacyResult"):
).geturl()
def _normalize_text_field(text: str, field_name: str, max_length: int) -> str:
if text and not isinstance(text, str):
text = str(text)
log.debug("result: invalid type of field '%s': %s", field_name, text)
text = WHITESPACE_REGEX.sub(" ", text).strip()
# truncate too long result titles and contents
if text and len(text) > max_length:
# don't split at characters, but at full words instead
# e.g. _truncate_text("hello world", 9) yields "hello" instead of "hello wor"
text = text[:max_length].rsplit(" ", 1)[0] + " …"
return text
def _normalize_text_fields(result: "MainResult | LegacyResult"):
# As soon we need LegacyResult not any longer, we can move this function to
@@ -101,18 +118,9 @@ def _normalize_text_fields(result: "MainResult | LegacyResult"):
# implemented correctly. Historically, however, we have always had a type
# check here.
if result.title and not isinstance(result.title, str):
log.debug("result: invalid type of field 'title': %s", str(result))
result.title = str(result)
if result.content and not isinstance(result.content, str):
log.debug("result: invalid type of field 'content': %s", str(result))
result.content = str(result)
result.title = _normalize_text_field(result.title, "title", MAXIMUM_TITLE_LENGTH_CHARS)
result.content = _normalize_text_field(result.content, "content", MAXIMUM_CONTENT_LENGTH_CHARS)
# normalize title and content
if result.title:
result.title = WHITESPACE_REGEX.sub(" ", result.title).strip()
if result.content:
result.content = WHITESPACE_REGEX.sub(" ", result.content).strip()
if result.content == result.title:
# avoid duplicate content between the content and title fields
result.content = ""