[fix] brave engines - web, images & videos (#5478)

brave web:
  xpath selectors needed to be justified

brave images & videos:
  The JS code with the JS object was read incorrectly; not always, but quite
  often, it led to exceptions when the Python data structure was created from it.

BTW: A complete review was conducted and corrections or additions were made to
the type definitions.

To test all brave engines in once::

    !br !brimg !brvid !brnews weather

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/ask.py

@@ -50,7 +50,7 @@ def response(resp):
     pos = script.index(end_tag) + len(end_tag) - 1
     script = script[:pos]
 
-    json_resp = utils.js_variable_to_python(script)
+    json_resp = utils.js_obj_str_to_python(script)
 
     results = []
 

searx/engines/brave.py

@@ -124,17 +124,17 @@ from urllib.parse import (
     urlparse,
 )
 
+import json
 from dateutil import parser
 from lxml import html
 
 from searx import locales
 from searx.utils import (
-    extr,
     extract_text,
-    eval_xpath,
     eval_xpath_list,
     eval_xpath_getindex,
-    js_variable_to_python,
+    js_obj_str_to_python,
+    js_obj_str_to_json_str,
     get_embeded_stream_url,
 )
 from searx.enginelib.traits import EngineTraits
@@ -142,17 +142,17 @@ from searx.result_types import EngineResults
 from searx.extended_types import SXNG_Response
 
 about = {
-    "website": 'https://search.brave.com/',
+    "website": "https://search.brave.com/",
-    "wikidata_id": 'Q22906900',
+    "wikidata_id": "Q22906900",
     "official_api_documentation": None,
     "use_official_api": False,
     "require_api_key": False,
-    "results": 'HTML',
+    "results": "HTML",
 }
 
 base_url = "https://search.brave.com/"
 categories = []
-brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = 'search'
+brave_category: t.Literal["search", "videos", "images", "news", "goggles"] = "search"
 """Brave supports common web-search, videos, images, news, and goggles search.
 
 - ``search``: Common WEB search
@@ -182,71 +182,86 @@ to do more won't return any result and you will most likely be flagged as a bot.
 """
 
 safesearch = True
-safesearch_map = {2: 'strict', 1: 'moderate', 0: 'off'}  # cookie: safesearch=off
+safesearch_map = {2: "strict", 1: "moderate", 0: "off"}  # cookie: safesearch=off
 
 time_range_support = False
 """Brave only supports time-range in :py:obj:`brave_category` ``search`` (UI
 category All) and in the goggles category."""
 
 time_range_map: dict[str, str] = {
-    'day': 'pd',
+    "day": "pd",
-    'week': 'pw',
+    "week": "pw",
-    'month': 'pm',
+    "month": "pm",
-    'year': 'py',
+    "year": "py",
 }
 
 
 def request(query: str, params: dict[str, t.Any]) -> None:
 
     args: dict[str, t.Any] = {
-        'q': query,
+        "q": query,
-        'source': 'web',
+        "source": "web",
     }
     if brave_spellcheck:
-        args['spellcheck'] = '1'
+        args["spellcheck"] = "1"
 
-    if brave_category in ('search', 'goggles'):
+    if brave_category in ("search", "goggles"):
-        if params.get('pageno', 1) - 1:
+        if params.get("pageno", 1) - 1:
-            args['offset'] = params.get('pageno', 1) - 1
+            args["offset"] = params.get("pageno", 1) - 1
-        if time_range_map.get(params['time_range']):
+        if time_range_map.get(params["time_range"]):
-            args['tf'] = time_range_map.get(params['time_range'])
+            args["tf"] = time_range_map.get(params["time_range"])
 
-    if brave_category == 'goggles':
+    if brave_category == "goggles":
-        args['goggles_id'] = Goggles
+        args["goggles_id"] = Goggles
 
     params["url"] = f"{base_url}{brave_category}?{urlencode(args)}"
+    logger.debug("url %s", params["url"])
 
     # set properties in the cookies
 
-    params['cookies']['safesearch'] = safesearch_map.get(params['safesearch'], 'off')
+    params["cookies"]["safesearch"] = safesearch_map.get(params["safesearch"], "off")
-    # the useLocation is IP based, we use cookie 'country' for the region
+    # the useLocation is IP based, we use cookie "country" for the region
-    params['cookies']['useLocation'] = '0'
+    params["cookies"]["useLocation"] = "0"
-    params['cookies']['summarizer'] = '0'
+    params["cookies"]["summarizer"] = "0"
 
-    engine_region = traits.get_region(params['searxng_locale'], 'all')
+    engine_region = traits.get_region(params["searxng_locale"], "all")
-    params['cookies']['country'] = engine_region.split('-')[-1].lower()  # type: ignore
+    params["cookies"]["country"] = engine_region.split("-")[-1].lower()  # type: ignore
 
-    ui_lang = locales.get_engine_locale(params['searxng_locale'], traits.custom["ui_lang"], 'en-us')
+    ui_lang = locales.get_engine_locale(params["searxng_locale"], traits.custom["ui_lang"], "en-us")
-    params['cookies']['ui_lang'] = ui_lang
+    params["cookies"]["ui_lang"] = ui_lang
-
+    logger.debug("cookies %s", params["cookies"])
-    logger.debug("cookies %s", params['cookies'])
-
-    params['headers']['Sec-Fetch-Dest'] = "document"
-    params['headers']['Sec-Fetch-Mode'] = "navigate"
-    params['headers']['Sec-Fetch-Site'] = "same-origin"
-    params['headers']['Sec-Fetch-User'] = "?1"
 
 
-def _extract_published_date(published_date_raw):
+def _extract_published_date(published_date_raw: str | None):
     if published_date_raw is None:
         return None
-
     try:
         return parser.parse(published_date_raw)
     except parser.ParserError:
         return None
 
 
+def extract_json_data(text: str) -> dict[str, t.Any]:
+    # Example script source containing the data:
+    #
+    # kit.start(app, element, {
+    #    node_ids: [0, 19],
+    #    data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]
+    #          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    text = text[text.index("<script") : text.index("</script")]
+    if not text:
+        raise ValueError("can't find JS/JSON data in the given text")
+    start = text.index("data: [{")
+    end = text.rindex("}}]")
+    js_obj_str = text[start:end]
+    js_obj_str = "{" + js_obj_str + "}}]}"
+    # js_obj_str = js_obj_str.replace("\xa0", "")  # remove ASCII for &nbsp;
+    # js_obj_str = js_obj_str.replace(r"\u003C", "<").replace(r"\u003c", "<")  # fix broken HTML tags in strings
+    json_str = js_obj_str_to_json_str(js_obj_str)
+    data: dict[str, t.Any] = json.loads(json_str)
+    return data
+
+
 def response(resp: SXNG_Response) -> EngineResults:
 
     if brave_category in ('search', 'goggles'):
@@ -261,11 +276,8 @@ def response(resp: SXNG_Response) -> EngineResults:
     #    node_ids: [0, 19],
     #    data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]
     #          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-    js_object = "[{" + extr(resp.text, "data: [{", "}}],") + "}}]"
+    json_data: dict[str, t.Any] = extract_json_data(resp.text)
-    json_data = js_variable_to_python(js_object)
+    json_resp: dict[str, t.Any] = json_data['data'][1]["data"]['body']['response']
-
-    # json_data is a list and at the second position (0,1) in this list we find the "response" data we need ..
-    json_resp = json_data[1]['data']['body']['response']
 
     if brave_category == 'images':
         return _parse_images(json_resp)
@@ -275,150 +287,121 @@ def response(resp: SXNG_Response) -> EngineResults:
     raise ValueError(f"Unsupported brave category: {brave_category}")
 
 
-def _parse_search(resp) -> EngineResults:
+def _parse_search(resp: SXNG_Response) -> EngineResults:
-    result_list = EngineResults()
+    res = EngineResults()
-
     dom = html.fromstring(resp.text)
 
-    # I doubt that Brave is still providing the "answer" class / I haven't seen
+    for result in eval_xpath_list(dom, "//div[contains(@class, 'snippet ')]"):
-    # answers in brave for a long time.
-    answer_tag = eval_xpath_getindex(dom, '//div[@class="answer"]', 0, default=None)
-    if answer_tag:
-        url = eval_xpath_getindex(dom, '//div[@id="featured_snippet"]/a[@class="result-header"]/@href', 0, default=None)
-        answer = extract_text(answer_tag)
-        if answer is not None:
-            result_list.add(result_list.types.Answer(answer=answer, url=url))
 
-    # xpath_results = '//div[contains(@class, "snippet fdb") and @data-type="web"]'
+        url: str | None = eval_xpath_getindex(result, ".//a/@href", 0, default=None)
-    xpath_results = '//div[contains(@class, "snippet ")]'
+        title_tag = eval_xpath_getindex(result, ".//div[contains(@class, 'title')]", 0, default=None)
-
-    for result in eval_xpath_list(dom, xpath_results):
-
-        url = eval_xpath_getindex(result, './/a[contains(@class, "h")]/@href', 0, default=None)
-        title_tag = eval_xpath_getindex(
-            result, './/a[contains(@class, "h")]//div[contains(@class, "title")]', 0, default=None
-        )
         if url is None or title_tag is None or not urlparse(url).netloc:  # partial url likely means it's an ad
             continue
 
-        content: str = extract_text(
+        content: str = ""
-            eval_xpath_getindex(result, './/div[contains(@class, "snippet-description")]', 0, default='')
+        pub_date = None
-        )  # type: ignore
-        pub_date_raw = eval_xpath(result, 'substring-before(.//div[contains(@class, "snippet-description")], "-")')
-        pub_date = _extract_published_date(pub_date_raw)
-        if pub_date and content.startswith(pub_date_raw):
-            content = content.lstrip(pub_date_raw).strip("- \n\t")
 
-        thumbnail = eval_xpath_getindex(result, './/img[contains(@class, "thumb")]/@src', 0, default='')
+        _content = eval_xpath_getindex(result, ".//div[contains(@class, 'content')]", 0, default="")
+        if len(_content):
+            content = extract_text(_content)  # type: ignore
+            _pub_date = extract_text(
+                eval_xpath_getindex(_content, ".//span[contains(@class, 't-secondary')]", 0, default="")
+            )
+            if _pub_date:
+                pub_date = _extract_published_date(_pub_date)
+                content = content.lstrip(_pub_date).strip("- \n\t")
 
-        item = {
+        thumbnail: str = eval_xpath_getindex(result, ".//a[contains(@class, 'thumbnail')]//img/@src", 0, default="")
-            'url': url,
+
-            'title': extract_text(title_tag),
+        item = res.types.LegacyResult(
-            'content': content,
+            template="default.html",
-            'publishedDate': pub_date,
+            url=url,
-            'thumbnail': thumbnail,
+            title=extract_text(title_tag),
-        }
+            content=content,
+            publishedDate=pub_date,
+            thumbnail=thumbnail,
+        )
+        res.add(item)
 
         video_tag = eval_xpath_getindex(
-            result, './/div[contains(@class, "video-snippet") and @data-macro="video"]', 0, default=None
+            result, ".//div[contains(@class, 'video-snippet') and @data-macro='video']", 0, default=[]
         )
-        if video_tag is not None:
+        if len(video_tag):
-
             # In my tests a video tag in the WEB search was most often not a
             # video, except the ones from youtube ..
-
             iframe_src = get_embeded_stream_url(url)
             if iframe_src:
-                item['iframe_src'] = iframe_src
+                item["iframe_src"] = iframe_src
-                item['template'] = 'videos.html'
+                item["template"] = "videos.html"
-                item['thumbnail'] = eval_xpath_getindex(video_tag, './/img/@src', 0, default='')
-                pub_date_raw = extract_text(
-                    eval_xpath(video_tag, './/div[contains(@class, "snippet-attributes")]/div/text()')
-                )
-                item['publishedDate'] = _extract_published_date(pub_date_raw)
-            else:
-                item['thumbnail'] = eval_xpath_getindex(video_tag, './/img/@src', 0, default='')
 
-        result_list.append(item)
+    return res
-
-    return result_list
 
 
-def _parse_news(resp) -> EngineResults:
+def _parse_news(resp: SXNG_Response) -> EngineResults:
-
+    res = EngineResults()
-    result_list = EngineResults()
     dom = html.fromstring(resp.text)
 
-    for result in eval_xpath_list(dom, '//div[contains(@class, "results")]//div[@data-type="news"]'):
+    for result in eval_xpath_list(dom, "//div[contains(@class, 'results')]//div[@data-type='news']"):
 
-        # import pdb
+        url = eval_xpath_getindex(result, ".//a[contains(@class, 'result-header')]/@href", 0, default=None)
-        # pdb.set_trace()
-
-        url = eval_xpath_getindex(result, './/a[contains(@class, "result-header")]/@href', 0, default=None)
         if url is None:
             continue
 
-        title = extract_text(eval_xpath_list(result, './/span[contains(@class, "snippet-title")]'))
+        title = eval_xpath_list(result, ".//span[contains(@class, 'snippet-title')]")
-        content = extract_text(eval_xpath_list(result, './/p[contains(@class, "desc")]'))
+        content = eval_xpath_list(result, ".//p[contains(@class, 'desc')]")
-        thumbnail = eval_xpath_getindex(result, './/div[contains(@class, "image-wrapper")]//img/@src', 0, default='')
+        thumbnail = eval_xpath_getindex(result, ".//div[contains(@class, 'image-wrapper')]//img/@src", 0, default="")
 
-        item = {
+        item = res.types.LegacyResult(
-            "url": url,
+            template="default.html",
-            "title": title,
+            url=url,
-            "content": content,
+            title=extract_text(title),
-            "thumbnail": thumbnail,
+            thumbnail=thumbnail,
-        }
+            content=extract_text(content),
+        )
+        res.add(item)
 
-        result_list.append(item)
+    return res
-
-    return result_list
 
 
-def _parse_images(json_resp) -> EngineResults:
+def _parse_images(json_resp: dict[str, t.Any]) -> EngineResults:
-    result_list = EngineResults()
+    res = EngineResults()
 
     for result in json_resp["results"]:
-        item = {
+        item = res.types.LegacyResult(
-            'url': result['url'],
+            template="images.html",
-            'title': result['title'],
+            url=result["url"],
-            'content': result['description'],
+            title=result["title"],
-            'template': 'images.html',
+            source=result["source"],
-            'resolution': result['properties']['format'],
+            img_src=result["properties"]["url"],
-            'source': result['source'],
+            thumbnail_src=result["thumbnail"]["src"],
-            'img_src': result['properties']['url'],
+        )
-            'thumbnail_src': result['thumbnail']['src'],
+        res.add(item)
-        }
-        result_list.append(item)
 
-    return result_list
+    return res
 
 
-def _parse_videos(json_resp) -> EngineResults:
+def _parse_videos(json_resp: dict[str, t.Any]) -> EngineResults:
-    result_list = EngineResults()
+    res = EngineResults()
 
     for result in json_resp["results"]:
-
+        item = res.types.LegacyResult(
-        url = result['url']
+            template="videos.html",
-        item = {
+            url=result["url"],
-            'url': url,
+            title=result["title"],
-            'title': result['title'],
+            content=result["description"],
-            'content': result['description'],
+            length=result["video"]["duration"],
-            'template': 'videos.html',
+            duration=result["video"]["duration"],
-            'length': result['video']['duration'],
+            publishedDate=_extract_published_date(result["age"]),
-            'duration': result['video']['duration'],
+        )
-            'publishedDate': _extract_published_date(result['age']),
+        if result["thumbnail"] is not None:
-        }
+            item["thumbnail"] = result["thumbnail"]["src"]
-
+        iframe_src = get_embeded_stream_url(result["url"])
-        if result['thumbnail'] is not None:
-            item['thumbnail'] = result['thumbnail']['src']
-
-        iframe_src = get_embeded_stream_url(url)
         if iframe_src:
-            item['iframe_src'] = iframe_src
+            item["iframe_src"] = iframe_src
 
-        result_list.append(item)
+        res.add(item)
 
-    return result_list
+    return res
 
 
 def fetch_traits(engine_traits: EngineTraits):
@@ -439,25 +422,25 @@ def fetch_traits(engine_traits: EngineTraits):
 
     resp = get('https://search.brave.com/settings')
 
-    if not resp.ok:  # type: ignore
+    if not resp.ok:
         print("ERROR: response from Brave is not OK.")
-    dom = html.fromstring(resp.text)  # type: ignore
+    dom = html.fromstring(resp.text)
 
-    for option in dom.xpath('//section//option[@value="en-us"]/../option'):
+    for option in dom.xpath("//section//option[@value='en-us']/../option"):
 
-        ui_lang = option.get('value')
+        ui_lang = option.get("value")
         try:
-            l = babel.Locale.parse(ui_lang, sep='-')
+            l = babel.Locale.parse(ui_lang, sep="-")
             if l.territory:
-                sxng_tag = region_tag(babel.Locale.parse(ui_lang, sep='-'))
+                sxng_tag = region_tag(babel.Locale.parse(ui_lang, sep="-"))
             else:
-                sxng_tag = language_tag(babel.Locale.parse(ui_lang, sep='-'))
+                sxng_tag = language_tag(babel.Locale.parse(ui_lang, sep="-"))
 
         except babel.UnknownLocaleError:
             print("ERROR: can't determine babel locale of Brave's (UI) language %s" % ui_lang)
             continue
 
-        conflict = engine_traits.custom["ui_lang"].get(sxng_tag)
+        conflict = engine_traits.custom["ui_lang"].get(sxng_tag)  # type: ignore
         if conflict:
             if conflict != ui_lang:
                 print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, ui_lang))
@@ -466,26 +449,26 @@ def fetch_traits(engine_traits: EngineTraits):
 
     # search regions of brave
 
-    resp = get('https://cdn.search.brave.com/serp/v2/_app/immutable/chunks/parameters.734c106a.js')
+    resp = get("https://cdn.search.brave.com/serp/v2/_app/immutable/chunks/parameters.734c106a.js")
 
-    if not resp.ok:  # type: ignore
+    if not resp.ok:
         print("ERROR: response from Brave is not OK.")
 
-    country_js = resp.text[resp.text.index("options:{all") + len('options:') :]  # type: ignore
+    country_js = resp.text[resp.text.index("options:{all") + len("options:") :]
     country_js = country_js[: country_js.index("},k={default")]
-    country_tags = js_variable_to_python(country_js)
+    country_tags = js_obj_str_to_python(country_js)
 
     for k, v in country_tags.items():
-        if k == 'all':
+        if k == "all":
-            engine_traits.all_locale = 'all'
+            engine_traits.all_locale = "all"
             continue
-        country_tag = v['value']
+        country_tag = v["value"]
 
         # add official languages of the country ..
         for lang_tag in babel.languages.get_official_languages(country_tag, de_facto=True):
             lang_tag = lang_map.get(lang_tag, lang_tag)
-            sxng_tag = region_tag(babel.Locale.parse('%s_%s' % (lang_tag, country_tag.upper())))
+            sxng_tag = region_tag(babel.Locale.parse("%s_%s" % (lang_tag, country_tag.upper())))
-            # print("%-20s: %s <-- %s" % (v['label'], country_tag, sxng_tag))
+            # print("%-20s: %s <-- %s" % (v["label"], country_tag, sxng_tag))
 
             conflict = engine_traits.regions.get(sxng_tag)
             if conflict:

searx/engines/duckduckgo.py

@@ -407,7 +407,7 @@ def fetch_traits(engine_traits: EngineTraits):
 
     """
     # pylint: disable=too-many-branches, too-many-statements, disable=import-outside-toplevel
-    from searx.utils import js_variable_to_python
+    from searx.utils import js_obj_str_to_python
 
     # fetch regions
 
@@ -455,7 +455,7 @@ def fetch_traits(engine_traits: EngineTraits):
 
     js_code = extr(resp.text, 'languages:', ',regions')  # type: ignore
 
-    languages = js_variable_to_python(js_code)
+    languages: dict[str, str] = js_obj_str_to_python(js_code)
     for eng_lang, name in languages.items():
 
         if eng_lang == 'wt_WT':

searx/engines/naver.py

@@ -15,7 +15,7 @@ from searx.utils import (
     extr,
     html_to_text,
     parse_duration_string,
-    js_variable_to_python,
+    js_obj_str_to_python,
     get_embeded_stream_url,
 )
 
@@ -125,7 +125,7 @@ def parse_images(data):
 
     match = extr(data, '<script>var imageSearchTabData=', '</script>')
     if match:
-        json = js_variable_to_python(match.strip())
+        json = js_obj_str_to_python(match.strip())
         items = json.get('content', {}).get('items', [])
 
         for item in items:

searx/engines/yandex.py

@@ -28,6 +28,20 @@ search_type = ""
 base_url_web = 'https://yandex.com/search/site/'
 base_url_images = 'https://yandex.com/images/search'
 
+# Supported languages
+yandex_supported_langs = [
+    "ru",  # Russian
+    "en",  # English
+    "be",  # Belarusian
+    "fr",  # French
+    "de",  # German
+    "id",  # Indonesian
+    "kk",  # Kazakh
+    "tt",  # Tatar
+    "tr",  # Turkish
+    "uk",  # Ukrainian
+]
+
 results_xpath = '//li[contains(@class, "serp-item")]'
 url_xpath = './/a[@class="b-serp-item__title-link"]/@href'
 title_xpath = './/h3[@class="b-serp-item__title"]/a[@class="b-serp-item__title-link"]/span'
@@ -48,6 +62,10 @@ def request(query, params):
         "searchid": "3131712",
     }
 
+    lang = params["language"].split("-")[0]
+    if lang in yandex_supported_langs:
+        query_params_web["lang"] = lang
+
     query_params_images = {
         "text": query,
         "uinfo": "sw-1920-sh-1080-ww-1125-wh-999",

searx/search/processors/online.py

@@ -155,8 +155,15 @@ class OnlineProcessor(EngineProcessor):
                     search_query.locale.language,
                 )
             headers["Accept-Language"] = ac_lang
-
         self.logger.debug("HTTP Accept-Language: %s", headers.get("Accept-Language", ""))
+
+        # https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header
+        headers["Sec-Fetch-Dest"] = "empty"
+        headers["Sec-Fetch-Mode"] = "cors"
+        headers["Sec-Fetch-Site"] = "same-origin"
+        headers["Sec-Fetch-User"] = "?1"
+        headers["Sec-GPC"] = "1"
+
         return params
 
     def _send_http_request(self, params: OnlineParams):

searx/settings.yml

@@ -2110,22 +2110,21 @@ engines:
     search_type: web
     shortcut: yd
     disabled: true
-    inactive: true
 
   - name: yandex images
     engine: yandex
+    network: yandex
     categories: images
     search_type: images
     shortcut: ydi
     disabled: true
-    inactive: true
 
   - name: yandex music
     engine: yandex_music
+    network: yandex
     shortcut: ydm
     disabled: true
     # https://yandex.com/support/music/access.html
-    inactive: true
 
   - name: yahoo
     engine: yahoo

searx/utils.py

@@ -49,9 +49,14 @@ _BLOCKED_TAGS = ('script', 'style')
 _ECMA_UNESCAPE4_RE = re.compile(r'%u([0-9a-fA-F]{4})', re.UNICODE)
 _ECMA_UNESCAPE2_RE = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)
 
-_JS_QUOTE_KEYS_RE = re.compile(r'([\{\s,])(\w+)(:)')
+_JS_STRING_DELIMITERS = re.compile(r'(["\'`])')
-_JS_VOID_RE = re.compile(r'void\s+[0-9]+|void\s*\([0-9]+\)')
+_JS_QUOTE_KEYS_RE = re.compile(r'([\{\s,])([\$_\w][\$_\w0-9]*)(:)')
-_JS_DECIMAL_RE = re.compile(r":\s*\.")
+_JS_VOID_OR_UNDEFINED_RE = re.compile(r'void\s+[0-9]+|void\s*\([0-9]+\)|undefined')
+_JS_DECIMAL_RE = re.compile(r"([\[\,:])\s*(\-?)\s*([0-9_]*)\.([0-9_]*)")
+_JS_DECIMAL2_RE = re.compile(r"([\[\,:])\s*(\-?)\s*([0-9_]+)")
+_JS_EXTRA_COMA_RE = re.compile(r"\s*,\s*([\]\}])")
+_JS_STRING_ESCAPE_RE = re.compile(r'\\(.)')
+_JSON_PASSTHROUGH_ESCAPES = R'"\bfnrtu'
 
 _XPATH_CACHE: dict[str, XPath] = {}
 _LANG_TO_LC_CACHE: dict[str, dict[str, str]] = {}
@@ -741,12 +746,53 @@ def detect_language(text: str, threshold: float = 0.3, only_search_languages: bo
     return None
 
 
-def js_variable_to_python(js_variable: str) -> t.Any:
+def _j2p_process_escape(match: re.Match[str]) -> str:
+    # deal with ECMA escape characters
+    _escape = match.group(1) or match.group(2)
+    return (
+        Rf'\{_escape}'
+        if _escape in _JSON_PASSTHROUGH_ESCAPES
+        else R'\u00' if _escape == 'x' else '' if _escape == '\n' else _escape
+    )
+
+
+def _j2p_decimal(match: re.Match[str]) -> str:
+    return (
+        match.group(1)
+        + match.group(2)
+        + (match.group(3).replace("_", "") or "0")
+        + "."
+        + (match.group(4).replace("_", "") or "0")
+    )
+
+
+def _j2p_decimal2(match: re.Match[str]) -> str:
+    return match.group(1) + match.group(2) + match.group(3).replace("_", "")
+
+
+def js_obj_str_to_python(js_obj_str: str) -> t.Any:
     """Convert a javascript variable into JSON and then load the value
 
     It does not deal with all cases, but it is good enough for now.
     chompjs has a better implementation.
     """
+    s = js_obj_str_to_json_str(js_obj_str)
+    # load the JSON and return the result
+    if s == "":
+        raise ValueError("js_obj_str can't be an empty string")
+    try:
+        return json.loads(s)
+    except json.JSONDecodeError as e:
+        logger.debug("Internal error: js_obj_str_to_python creates invalid JSON:\n%s", s)
+        raise ValueError("js_obj_str_to_python creates invalid JSON") from e
+
+
+def js_obj_str_to_json_str(js_obj_str: str) -> str:
+    if not isinstance(js_obj_str, str):
+        raise ValueError("js_obj_str must be of type str")
+    if js_obj_str == "":
+        raise ValueError("js_obj_str can't be an empty string")
+
     # when in_string is not None, it contains the character that has opened the string
     # either simple quote or double quote
     in_string = None
@@ -754,61 +800,78 @@ def js_variable_to_python(js_variable: str) -> t.Any:
     # r"""{ a:"f\"irst", c:'sec"ond'}"""
     # becomes
     # ['{ a:', '"', 'f\\', '"', 'irst', '"', ', c:', "'", 'sec', '"', 'ond', "'", '}']
-    parts = re.split(r'(["\'])', js_variable)
+    parts = _JS_STRING_DELIMITERS.split(js_obj_str)
-    # previous part (to check the escape character antislash)
+    # does the previous part ends with a backslash?
-    previous_p = ""
+    blackslash_just_before = False
     for i, p in enumerate(parts):
-        # parse characters inside a ECMA string
+        if p == in_string and not blackslash_just_before:
-        if in_string:
+            # * the current part matches the character which has opened the string
-            # we are in a JS string: replace the colon by a temporary character
+            # * there is no antislash just before
-            # so quote_keys_regex doesn't have to deal with colon inside the JS strings
+            # --> the current part close the current string
-            parts[i] = parts[i].replace(':', chr(1))
-            if in_string == "'":
-                # the JS string is delimited by simple quote.
-                # This is not supported by JSON.
-                # simple quote delimited string are converted to double quote delimited string
-                # here, inside a JS string, we escape the double quote
-                parts[i] = parts[i].replace('"', r'\"')
-
-        # deal with delimiters and escape character
-        if not in_string and p in ('"', "'"):
-            # we are not in string
-            # but p is double or simple quote
-            # that's the start of a new string
-            # replace simple quote by double quote
-            # (JSON doesn't support simple quote)
-            parts[i] = '"'
-            in_string = p
-            continue
-        if p == in_string:
-            # we are in a string and the current part MAY close the string
-            if len(previous_p) > 0 and previous_p[-1] == '\\':
-                # there is an antislash just before: the ECMA string continue
-                continue
-            # the current p close the string
-            # replace simple quote by double quote
-            parts[i] = '"'
             in_string = None
+            # replace simple quote and ` by double quote
+            # since JSON supports only double quote for string
+            parts[i] = '"'
 
-        if not in_string:
+        elif in_string:
-            # replace void 0 by null
+            # --> we are in a JS string
+            # replace the colon by a temporary character
+            # so _JS_QUOTE_KEYS_RE doesn't have to deal with colon inside the JS strings
+            p = p.replace(':', chr(1))
+            # replace JS escape sequences by JSON escape sequences
+            p = _JS_STRING_ESCAPE_RE.sub(_j2p_process_escape, p)
+            # the JS string is delimited by simple quote.
+            # This is not supported by JSON.
+            # simple quote delimited string are converted to double quote delimited string
+            # here, inside a JS string, we escape the double quote
+            if in_string == "'":
+                p = p.replace('"', r'\"')
+            parts[i] = p
+            # deal with the sequence blackslash then quote
+            # since js_obj_str splits on quote, we detect this case:
+            # * the previous part ends with a black slash
+            # * the current part is a single quote
+            # when detected the blackslash is removed on the previous part
+            if blackslash_just_before and p[:1] == "'":
+                parts[i - 1] = parts[i - 1][:-1]
+
+        elif in_string is None and p in ('"', "'", "`"):
+            # we are not in string but p is string delimiter
+            # --> that's the start of a new string
+            in_string = p
+            # replace simple quote by double quote
+            # since JSON supports only double quote for string
+            parts[i] = '"'
+
+        elif in_string is None:
+            # we are not in a string
+            # replace by null these values:
+            # * void 0
+            # * void(0)
+            # * undefined
             # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/void
-            # we are sure there is no string in p
+            p = _JS_VOID_OR_UNDEFINED_RE.sub("null", p)
-            parts[i] = _JS_VOID_RE.sub("null", p)
+            # make sure there is a leading zero in front of float
-        # update previous_p
+            p = _JS_DECIMAL_RE.sub(_j2p_decimal, p)
-        previous_p = p
+            p = _JS_DECIMAL2_RE.sub(_j2p_decimal2, p)
+            # remove extra coma in a list or an object
+            # for example [1,2,3,] becomes [1,2,3]
+            p = _JS_EXTRA_COMA_RE.sub(lambda match: match.group(1), p)
+            parts[i] = p
+
+        # update for the next iteration
+        blackslash_just_before = len(p) > 0 and p[-1] == '\\'
+
     # join the string
     s = ''.join(parts)
-    # add quote around the key
+    # add quote arround the key
     # { a: 12 }
     # becomes
     # { "a": 12 }
     s = _JS_QUOTE_KEYS_RE.sub(r'\1"\2"\3', s)
-    s = _JS_DECIMAL_RE.sub(":0.", s)
+    # replace the surogate character by colon and strip whitespaces
-    # replace the surogate character by colon
+    s = s.replace(chr(1), ':').strip()
-    s = s.replace(chr(1), ':')
+    return s
-    # load the JSON and return the result
-    return json.loads(s)
 
 
 def parse_duration_string(duration_str: str) -> timedelta | None:

tests/unit/test_js_variable_to_python.py

@@ -0,0 +1,286 @@
+# -*- coding: utf-8 -*-
+"""Tests for the function ``searx.utils.js_obj_str_to_python``
+
+The tests are copied from:
+
+https://github.com/Nykakin/chompjs/blob/c1501b5cd82c0044539875331745b820e7bfd067/chompjs/test_parser.py
+
+The commented-out tests are not yet supported by the current implementation.
+"""
+# pylint: disable=missing-class-docstring, invalid-name
+
+import math
+
+from parameterized import parameterized
+
+from searx.utils import js_obj_str_to_python
+
+from tests import SearxTestCase
+
+
+class TestParser(SearxTestCase):
+    @parameterized.expand(
+        [
+            ("{'hello': 'world'}", {'hello': 'world'}),
+            ("{'hello': 'world', 'my': 'master'}", {'hello': 'world', 'my': 'master'}),
+            (
+                "{'hello': 'world', 'my': {'master': 'of Orion'}, 'test': 'xx'}",
+                {'hello': 'world', 'my': {'master': 'of Orion'}, 'test': 'xx'},
+            ),
+            ("{}", {}),
+        ]
+    )
+    def test_parse_object(self, js, expected_py):
+        py = js_obj_str_to_python(js)
+        self.assertEqual(py, expected_py)
+
+    @parameterized.expand(
+        [
+            ("[]", []),
+            ("[[[]]]", [[[]]]),
+            ("[[[1]]]", [[[1]]]),
+            ("[1]", [1]),
+            ("[1, 2, 3, 4]", [1, 2, 3, 4]),
+            ("['h', 'e', 'l', 'l', 'o']", ['h', 'e', 'l', 'l', 'o']),
+            ("[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]", [[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]),
+        ]
+    )
+    def test_parse_list(self, js, expected_py):
+        py = js_obj_str_to_python(js)
+        self.assertEqual(py, expected_py)
+
+    @parameterized.expand(
+        [
+            ("{'hello': [], 'world': [0]}", {'hello': [], 'world': [0]}),
+            ("{'hello': [1, 2, 3, 4]}", {'hello': [1, 2, 3, 4]}),
+            ("[{'a':12}, {'b':33}]", [{'a': 12}, {'b': 33}]),
+            (
+                "[false, {'true': true, `pies`: \"kot\"}, false,]",
+                [False, {"true": True, 'pies': 'kot'}, False],
+            ),
+            (
+                "{a:1,b:1,c:1,d:1,e:1,f:1,g:1,h:1,i:1,j:1}",
+                {k: 1 for k in 'abcdefghij'},
+            ),
+            (
+                "{'a':[{'b':1},{'c':[{'d':{'f':{'g':[1,2]}}},{'e':1}]}]}",
+                {'a': [{'b': 1}, {'c': [{'d': {'f': {'g': [1, 2]}}}, {'e': 1}]}]},
+            ),
+        ]
+    )
+    def test_parse_mixed(self, js, expected_py):
+        py = js_obj_str_to_python(js)
+        self.assertEqual(py, expected_py)
+
+    @parameterized.expand(
+        [
+            ("{'hello': 12, 'world': 10002.21}", {'hello': 12, 'world': 10002.21}),
+            ("[12, -323, 0.32, -32.22, .2, - 4]", [12, -323, 0.32, -32.22, 0.2, -4]),
+            ('{"a": -12, "b": - 5}', {'a': -12, 'b': -5}),
+            ("{'a': true, 'b': false, 'c': null}", {'a': True, 'b': False, 'c': None}),
+            ("[\"\\uD834\\uDD1E\"]", ['𝄞']),
+            ("{'a': '123\\'456\\n'}", {'a': "123'456\n"}),
+            ("['\u00e9']", ['é']),
+            ('{"cache":{"\u002ftest\u002f": 0}}', {'cache': {'/test/': 0}}),
+            ('{"a": 3.125e7}', {'a': 3.125e7}),
+            ('''{"a": "b\\'"}''', {'a': "b'"}),
+            ('{"a": .99, "b": -.1}', {"a": 0.99, "b": -0.1}),
+            ('["/* ... */", "// ..."]', ["/* ... */", "// ..."]),
+            ('{"inclusions":["/*","/"]}', {'inclusions': ['/*', '/']}),
+        ]
+    )
+    def test_parse_standard_values(self, js, expected_py):
+        py = js_obj_str_to_python(js)
+        self.assertEqual(py, expected_py)
+
+    def test_parse_nan(self):
+        js = '{"A": NaN}'
+        py = js_obj_str_to_python(js)
+        self.assertTrue(math.isnan(py["A"]))
+
+    @parameterized.expand(
+        [
+            ("{abc: 100, dev: 200}", {'abc': 100, 'dev': 200}),
+            ("{abcdefghijklmnopqrstuvwxyz: 12}", {"abcdefghijklmnopqrstuvwxyz": 12}),
+            # (
+            #     "{age: function(yearBorn,thisYear) {return thisYear - yearBorn;}}",
+            #     {"age": "function(yearBorn,thisYear) {return thisYear - yearBorn;}"}
+            # ),
+            # (
+            #     "{\"abc\": function() {return '])))))))))))))))';}}",
+            #     {"abc": "function() {return '])))))))))))))))';}"},
+            # ),
+            ('{"a": undefined}', {"a": None}),  # chompjs returns {"a": "undefined"}
+            ('[undefined, undefined]', [None, None]),  # chompjs returns ["undefined", "undefined"]
+            ("{_a: 1, $b: 2}", {"_a": 1, "$b": 2}),
+            # ("{regex: /a[^d]{1,12}/i}", {'regex': '/a[^d]{1,12}/i'}),
+            # ("{'a': function(){return '\"'}}", {'a': 'function(){return \'"\'}'}),
+            ("{1: 1, 2: 2, 3: 3, 4: 4}", {'1': 1, '2': 2, '3': 3, '4': 4}),
+            ("{'a': 121.}", {'a': 121.0}),
+        ]
+    )
+    def test_parse_strange_values(self, js, expected_py):
+        py = js_obj_str_to_python(js)
+        self.assertEqual(py, expected_py)
+
+    @parameterized.expand(
+        [
+            # ('{"a": {"b": [12, 13, 14]}}text text', {"a": {"b": [12, 13, 14]}}),
+            # ('var test = {"a": {"b": [12, 13, 14]}}', {"a": {"b": [12, 13, 14]}}),
+            ('{"a":\r\n10}', {'a': 10}),
+            ("{'foo': 0,\r\n}", {'foo': 0}),
+            ("{truefalse: 0, falsefalse: 1, nullnull: 2}", {'truefalse': 0, 'falsefalse': 1, 'nullnull': 2}),
+        ]
+    )
+    def test_strange_input(self, js, expected_py):
+        py = js_obj_str_to_python(js)
+        self.assertEqual(py, expected_py)
+
+    @parameterized.expand(
+        [
+            ("[0]", [0]),
+            ("[1]", [1]),
+            ("[12]", [12]),
+            ("[12_12]", [1212]),
+            # ("[0x12]", [18]),
+            # ("[0xab]", [171]),
+            # ("[0xAB]", [171]),
+            # ("[0X12]", [18]),
+            # ("[0Xab]", [171]),
+            # ("[0XAB]", [171]),
+            # ("[01234]", [668]),
+            # ("[0o1234]", [668]),
+            # ("[0O1234]", [668]),
+            # ("[0b1111]", [15]),
+            # ("[0B1111]", [15]),
+            ("[-0]", [-0]),
+            ("[-1]", [-1]),
+            ("[-12]", [-12]),
+            ("[-12_12]", [-1212]),
+            # ("[-0x12]", [-18]),
+            # ("[-0xab]", [-171]),
+            # ("[-0xAB]", [-171]),
+            # ("[-0X12]", [-18]),
+            # ("[-0Xab]", [-171]),
+            # ("[-0XAB]", [-171]),
+            # ("[-01234]", [-668]),
+            # ("[-0o1234]", [-668]),
+            # ("[-0O1234]", [-668]),
+            # ("[-0b1111]", [-15]),
+            # ("[-0B1111]", [-15]),
+        ]
+    )
+    def test_integer_numeric_values(self, js, expected_py):
+        py = js_obj_str_to_python(js)
+        self.assertEqual(py, expected_py)
+
+    @parameterized.expand(
+        [
+            ("[0.32]", [0.32]),
+            ("[-0.32]", [-0.32]),
+            ("[.32]", [0.32]),
+            ("[-.32]", [-0.32]),
+            ("[12.]", [12.0]),
+            ("[-12.]", [-12.0]),
+            ("[12.32]", [12.32]),
+            ("[-12.12]", [-12.12]),
+            ("[3.1415926]", [3.1415926]),
+            ("[.123456789]", [0.123456789]),
+            ("[.0123]", [0.0123]),
+            ("[0.0123]", [0.0123]),
+            ("[-.0123]", [-0.0123]),
+            ("[-0.0123]", [-0.0123]),
+            ("[3.1E+12]", [3.1e12]),
+            ("[3.1e+12]", [3.1e12]),
+            ("[.1e-23]", [0.1e-23]),
+            ("[.1e-23]", [0.1e-23]),
+        ]
+    )
+    def test_float_numeric_values(self, js, expected_py):
+        py = js_obj_str_to_python(js)
+        self.assertEqual(py, expected_py)
+
+    # @parameterized.expand([
+    #     ('["Test\\nDrive"]\n{"Test": "Drive"}', [['Test\nDrive'], {'Test': 'Drive'}]),
+    # ])
+    # def test_jsonlines(self, js, expected_py):
+    #     py = js_obj_str_to_python(js)
+    #     self.assertEqual(py, expected_py)
+
+
+class TestParserExceptions(SearxTestCase):
+    @parameterized.expand(
+        [
+            ('}{', ValueError),
+            ('', ValueError),
+            (None, ValueError),
+        ]
+    )
+    def test_exceptions(self, js, expected_exception):
+        with self.assertRaises(expected_exception):
+            js_obj_str_to_python(js)
+
+    @parameterized.expand(
+        [
+            ("{whose: 's's', category_name: '>'}", ValueError),
+        ]
+    )
+    def test_malformed_input(self, in_data, expected_exception):
+        with self.assertRaises(expected_exception):
+            js_obj_str_to_python(in_data)
+
+    @parameterized.expand(
+        [
+            (
+                '{"test": """}',
+                ValueError,
+                'js_obj_str_to_python creates invalid JSON',
+            ),
+        ]
+    )
+    def test_error_messages(self, js, expected_exception, expected_exception_text):
+        with self.assertRaisesRegex(expected_exception, expected_exception_text):
+            js_obj_str_to_python(js)
+
+
+# class TestOptions(SearxTestCase):
+#     @parameterized.expand(
+#         [
+#             ('{\\\"a\\\": 12}', {'a': 12}),
+#         ]
+#     )
+#     def test_unicode_escape(self, js, expected_py):
+#         py = js_obj_str_to_python(js)
+#         self.assertEqual(py, expected_py)
+
+
+class TestParseJsonObjects(SearxTestCase):
+    @parameterized.expand(
+        [
+            # ("", []),
+            # ("aaaaaaaaaaaaaaaa", []),
+            # ("         ", []),
+            ("      {'a': 12}", [{'a': 12}]),
+            # ("[1, 2, 3, 4]xxxxxxxxxxxxxxxxxxxxxxxx", [[1, 2, 3, 4]]),
+            # ("[12] [13] [14]", [[12], [13], [14]]),
+            # ("[10] {'a': [1, 1, 1,]}", [[10], {'a': [1, 1, 1]}]),
+            # ("[1][1][1]", [[1], [1], [1]]),
+            # ("[1] [2] {'a': ", [[1], [2]]),
+            # ("[]", [[]]),
+            # ("[][][][]", [[], [], [], []]),
+            ("{}", [{}]),
+            # ("{}{}{}{}", [{}, {}, {}, {}]),
+            # ("{{}}{{}}", []),
+            # ("[[]][[]]", [[[]], [[]]]),
+            # ("{am: 'ab'}\n{'ab': 'xx'}", [{'am': 'ab'}, {'ab': 'xx'}]),
+            # (
+            #     'function(a, b, c){ /* ... */ }({"a": 12}, Null, [1, 2, 3])',
+            #     [{}, {'a': 12}, [1, 2, 3]],
+            # ),
+            # ('{"a": 12, broken}{"c": 100}', [{'c': 100}]),
+            # ('[12,,,,21][211,,,][12,12][12,,,21]', [[12, 12]]),
+        ]
+    )
+    def test_parse_json_objects(self, js, expected_py):
+        py_in_list = [js_obj_str_to_python(js)]
+        self.assertEqual(py_in_list, expected_py)