[fix] naver: update HTML parsing for redesigned Naver search results page

Naver redesigned their search results page, replacing the old CSS classes
(lst_total, link_tit, total_dsc_wrap, api_txt_lines) with a new layout
using fds-web-doc-root containers and sds-comps design system.

Updated parse_general():
- Result container: fds-web-doc-root + fds-web-normal-doc-root
- Title: sds-comps-text-type-headline1 span
- URL: first external http link (with try/except guard)
- Content: sds-comps-text-type-body1 span
- Thumbnail: profile-thumbnail img src
- Added guard: skip results without title or URL

searx/engines/baidu.py

@@ -7,13 +7,17 @@
 # There exits a https://github.com/ohblue/baidu-serp-api/
 # but we don't use it here (may we can learn from).
 
+import typing as t
+
 from urllib.parse import urlencode
 from datetime import datetime
 from html import unescape
 import time
 import json
 
-from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
+from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException, SearxEngineAccessDeniedException
+from searx.enginelib import EngineCache
+from searx.network import get as http_get
 from searx.utils import html_to_text
 
 about = {
@@ -35,6 +39,31 @@ baidu_category = 'general'
 time_range_support = True
 time_range_dict = {"day": 86400, "week": 604800, "month": 2592000, "year": 31536000}
 
+image_base_url = "https://image.baidu.com/"
+
+COOKIE_CACHE_KEY = "cookie"
+COOKIE_CACHE_EXPIRATION_SECONDS = 3600
+
+CACHE: EngineCache
+"""Stores cookies from Baidu image search warmup."""
+
+
+def setup(engine_settings: dict[str, t.Any]) -> bool:
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])
+    return True
+
+
+def get_image_cookies(headers: dict[str, str]) -> dict[str, str]:
+    cookies: dict[str, str] | None = CACHE.get(COOKIE_CACHE_KEY)
+    if cookies:
+        return cookies
+
+    warmup = http_get(image_base_url, headers=headers, timeout=10)
+    cookies = dict(warmup.cookies.items())
+    CACHE.set(key=COOKIE_CACHE_KEY, value=cookies, expire=COOKIE_CACHE_EXPIRATION_SECONDS)
+    return cookies
+
 
 def init(_):
     if baidu_category not in ('general', 'images', 'it'):
@@ -88,6 +117,9 @@ def request(query, params):
         if baidu_category == 'it':
             query_params["paramList"] += f",timestamp_range={past}-{now}"
 
+    if baidu_category == 'images':
+        params["cookies"] = get_image_cookies(params["headers"])
+
     params["url"] = f"{query_url}?{urlencode(query_params)}"
     params["allow_redirects"] = False
     return params
@@ -103,6 +135,8 @@ def response(resp):
         # baidu's JSON encoder wrongly quotes / and ' characters by \\ and \'
         text = text.replace(r"\/", "/").replace(r"\'", "'")
     data = json.loads(text, strict=False)
+    if data.get("antiFlag") == 1:
+        raise SearxEngineAccessDeniedException(data.get("message", "Forbid spider access"))
     parsers = {'general': parse_general, 'images': parse_images, 'it': parse_it}
 
     return parsers[baidu_category](data)

searx/engines/bilibili.py

@@ -8,6 +8,7 @@ import random
 import string
 from urllib.parse import urlencode
 from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
 
 from searx import utils
 
@@ -39,6 +40,32 @@ cookie = {
     "home_feed_column": "4",
 }
 
+_CN_TZ = ZoneInfo("Asia/Shanghai")
+
+# Calendar-day time filter (Asia/Shanghai); dict values are days to subtract from today.
+time_range_support = True
+time_range_dict = {"day": 0, "week": 6, "month": 29, "year": 364}
+
+
+def _pubtime_range(time_range: str) -> tuple[int, int]:
+    """Return ``(pubtime_begin_s, pubtime_end_s)`` for Bilibili's search API.
+
+    Time ranges follow Bilibili's website semantics: they are counted in
+    **calendar days** in China Standard Time (``Asia/Shanghai``), not as
+    sliding 24-hour windows.  For example, ``day`` means from 00:00:00 to
+    23:59:59 of the current local day; ``week`` spans from 00:00:00 on the
+    calendar day six days ago through the end of today, and so on.
+
+    The returned Unix timestamps (seconds) map to Bilibili's
+    ``pubtime_begin_s`` and ``pubtime_end_s`` query parameters.
+    """
+    now = datetime.now(_CN_TZ)
+    pubtime_end_s = int(now.replace(hour=23, minute=59, second=59, microsecond=0).timestamp())
+    begin_day = now - timedelta(days=time_range_dict[time_range])
+    pubtime_begin_s = int(begin_day.replace(hour=0, minute=0, second=0, microsecond=0).timestamp())
+
+    return pubtime_begin_s, pubtime_end_s
+
 
 def request(query, params):
     query_params = {
@@ -50,6 +77,11 @@ def request(query, params):
         "search_type": "video",
     }
 
+    if params.get("time_range") in time_range_dict:
+        pubtime_begin_s, pubtime_end_s = _pubtime_range(params["time_range"])
+        query_params["pubtime_begin_s"] = pubtime_begin_s
+        query_params["pubtime_end_s"] = pubtime_end_s
+
     params["url"] = f"{base_url}?{urlencode(query_params)}"
     params["headers"]["Referer"] = "https://www.bilibili.com/"
     params["headers"]["Accept"] = "application/json, text/javascript, */*; q=0.01"

searx/engines/naver.py

@@ -99,23 +99,35 @@ def parse_general(data):
 
     dom = html.fromstring(data)
 
-    for item in eval_xpath_list(dom, "//ul[contains(@class, 'lst_total')]/li[contains(@class, 'bx')]"):
+    for item in eval_xpath_list(dom, "//div[contains(@class, 'fds-web-normal-doc-root')]"):
-        thumbnail = None
+        thumbnail = extract_text(
+            eval_xpath(
+                item,
+                ".//div[contains(@class, 'sds-comps-image') and not(contains(@class, 'sds-comps-image-circle'))]/img/@src",
+            )
+        )
+
+        title = extract_text(eval_xpath(item, ".//span[contains(@class, 'sds-comps-text-type-headline1')]"))
+
+        url = None
         try:
-            thumbnail = eval_xpath_getindex(item, ".//div[contains(@class, 'thumb_single')]//img/@data-lazysrc", 0)
+            url = eval_xpath_getindex(
+                item, ".//a[starts-with(@href, 'http') and not(contains(@href, 'keep.naver.com'))]/@href", 0
+            )
         except (ValueError, TypeError, SearxEngineXPathException):
             pass
 
-        results.add(
+        content = extract_text(eval_xpath(item, ".//span[contains(@class, 'sds-comps-text-type-body1')]"))
-            MainResult(
+
-                title=extract_text(eval_xpath(item, ".//a[contains(@class, 'link_tit')]")),
+        if title and url:
-                url=eval_xpath_getindex(item, ".//a[contains(@class, 'link_tit')]/@href", 0),
+            results.add(
-                content=extract_text(
+                MainResult(
-                    eval_xpath(item, ".//div[contains(@class, 'total_dsc_wrap')]//a[contains(@class, 'api_txt_lines')]")
+                    title=title,
-                ),
+                    url=url,
-                thumbnail=thumbnail,
+                    content=content or "",
+                    thumbnail=thumbnail or "",
+                )
             )
-        )
 
     return results
 
@@ -173,7 +185,7 @@ def parse_news(data):
                     title=title,
                     url=url,
                     content=content,
-                    thumbnail=thumbnail,
+                    thumbnail=thumbnail or "",
                 )
             )
 
@@ -196,7 +208,7 @@ def parse_videos(data):
 
         length = None
         try:
-            length = parse_duration_string(extract_text(eval_xpath(item, ".//span[contains(@class, 'time')]")))
+            length = parse_duration_string(extract_text(eval_xpath(item, ".//span[contains(@class, 'time')]")) or "")
         except (ValueError, TypeError):
             pass
 

searx/engines/neosearch.py

@@ -48,6 +48,9 @@ def response(resp: "SXNG_Response") -> EngineResults:
     for lens in json_resp["lenses"]:
         for category in lens["categories"]:
             for result in category["links"]:
+                if not result["url"]:
+                    continue
+
                 res.add(
                     res.types.MainResult(
                         url=result["url"],