[feat] engines: add Naver engine (#4573)

Refactor Naver engine (Web, News, Images, Videos, Autocomplete)

- ref: https://search.naver.com/
- lang: `ko`
- Wikidata: https://www.wikidata.org/wiki/Q485639

Co-authored-by: Bnyro <bnyro@tutanota.com>

container/entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 # shellcheck shell=dash
-set -eu
+set -u
 
 check_file() {
     local target="$1"

docs/admin/settings/settings_search.rst

@@ -41,6 +41,7 @@
   - ``duckduckgo``
   - ``google``
   - ``mwmbl``
+  - ``naver``
   - ``quark``
   - ``qwant``
   - ``seznam``

searx/autocomplete.py

@@ -149,6 +149,21 @@ def mwmbl(query, _lang):
     return [result for result in results if not result.startswith("go: ") and not result.startswith("search: ")]
 
 
+def naver(query, _lang):
+    # Naver search autocompleter
+    url = f"https://ac.search.naver.com/nx/ac?{urlencode({'q': query, 'r_format': 'json', 'st': 0})}"
+    response = get(url)
+
+    results = []
+
+    if response.ok:
+        data = response.json()
+        if data.get('items'):
+            for item in data['items'][0]:
+                results.append(item[0])
+    return results
+
+
 def qihu360search(query, _lang):
     # 360Search search autocompleter
     url = f"https://sug.so.360.cn/suggest?{urlencode({'format': 'json', 'word': query})}"
@@ -300,6 +315,7 @@ backends = {
     'duckduckgo': duckduckgo,
     'google': google_complete,
     'mwmbl': mwmbl,
+    'naver': naver,
     'quark': quark,
     'qwant': qwant,
     'seznam': seznam,

searx/engines/material_icons.py

@@ -1,7 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Material Icons (images)
-
-"""
+"""Material Icons (icons)"""
 
 import re
 from json import loads
@@ -14,6 +12,8 @@ about = {
     "require_api_key": False,
     "results": 'JSON',
 }
+categories = ['images', 'icons']
+
 search_url = "https://fonts.google.com/metadata/icons?key=material_symbols&incomplete=true"
 result_url = "https://fonts.google.com/icons?icon.query={query}&selected=Material+Symbols+Outlined:{icon_name}:FILL@0{fill};wght@400;GRAD@0;opsz@24"  # pylint: disable=line-too-long
 img_src_url = "https://fonts.gstatic.com/s/i/short-term/release/materialsymbolsoutlined/{icon_name}/{svg_type}/24px.svg"
@@ -46,7 +46,7 @@ def response(resp):
             continue
 
         tags = [tag.title() for tag in result["tags"]]
-        categories = [category.title() for category in result["categories"]]
+        icon_categories = [category.title() for category in result["categories"]]
 
         results.append(
             {
@@ -54,7 +54,7 @@ def response(resp):
                 'url': result_url.format(icon_name=result["name"], query=result["name"], fill=0 if outlined else 1),
                 'img_src': img_src_url.format(icon_name=result["name"], svg_type=svg_type),
                 'title': result["name"].replace("_", "").title(),
-                'content': ", ".join(tags) + " / " + ", ".join(categories),
+                'content': ", ".join(tags) + " / " + ", ".join(icon_categories),
             }
         )
 

searx/engines/naver.py

@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=line-too-long
+"""Naver for SearXNG"""
+
+from urllib.parse import urlencode
+from lxml import html
+
+from searx.exceptions import SearxEngineAPIException, SearxEngineXPathException
+from searx.result_types import EngineResults, MainResult
+from searx.utils import (
+    eval_xpath_getindex,
+    eval_xpath_list,
+    eval_xpath,
+    extract_text,
+    extr,
+    html_to_text,
+    parse_duration_string,
+    js_variable_to_python,
+)
+
+# engine metadata
+about = {
+    "website": "https://search.naver.com",
+    "wikidata_id": "Q485639",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+    "language": "ko",
+}
+
+categories = []
+paging = True
+
+time_range_support = True
+time_range_dict = {"day": "1d", "week": "1w", "month": "1m", "year": "1y"}
+
+base_url = "https://search.naver.com"
+
+naver_category = "general"
+"""Naver supports general, images, news, videos search.
+
+- ``general``: search for general
+- ``images``: search for images
+- ``news``: search for news
+- ``videos``: search for videos
+"""
+
+# Naver cannot set the number of results on one page, set default value for paging
+naver_category_dict = {
+    "general": {
+        "start": 15,
+        "where": "web",
+    },
+    "images": {
+        "start": 50,
+        "where": "image",
+    },
+    "news": {
+        "start": 10,
+        "where": "news",
+    },
+    "videos": {
+        "start": 48,
+        "where": "video",
+    },
+}
+
+
+def init(_):
+    if naver_category not in ('general', 'images', 'news', 'videos'):
+        raise SearxEngineAPIException(f"Unsupported category: {naver_category}")
+
+
+def request(query, params):
+    query_params = {
+        "query": query,
+    }
+
+    if naver_category in naver_category_dict:
+        query_params["start"] = (params["pageno"] - 1) * naver_category_dict[naver_category]["start"] + 1
+        query_params["where"] = naver_category_dict[naver_category]["where"]
+
+    if params["time_range"] in time_range_dict:
+        query_params["nso"] = f"p:{time_range_dict[params['time_range']]}"
+
+    params["url"] = f"{base_url}/search.naver?{urlencode(query_params)}"
+    return params
+
+
+def response(resp) -> EngineResults:
+    parsers = {'general': parse_general, 'images': parse_images, 'news': parse_news, 'videos': parse_videos}
+
+    return parsers[naver_category](resp.text)
+
+
+def parse_general(data):
+    results = EngineResults()
+
+    dom = html.fromstring(data)
+
+    for item in eval_xpath_list(dom, "//ul[contains(@class, 'lst_total')]/li[contains(@class, 'bx')]"):
+        thumbnail = None
+        try:
+            thumbnail = eval_xpath_getindex(item, ".//div[contains(@class, 'thumb_single')]//img/@data-lazysrc", 0)
+        except (ValueError, TypeError, SearxEngineXPathException):
+            pass
+
+        results.add(
+            MainResult(
+                title=extract_text(eval_xpath(item, ".//a[contains(@class, 'link_tit')]")),
+                url=eval_xpath_getindex(item, ".//a[contains(@class, 'link_tit')]/@href", 0),
+                content=extract_text(
+                    eval_xpath(item, ".//div[contains(@class, 'total_dsc_wrap')]//a[contains(@class, 'api_txt_lines')]")
+                ),
+                thumbnail=thumbnail,
+            )
+        )
+
+    return results
+
+
+def parse_images(data):
+    results = []
+
+    match = extr(data, '<script>var imageSearchTabData=', '</script>')
+    if match:
+        json = js_variable_to_python(match.strip())
+        items = json.get('content', {}).get('items', [])
+
+        for item in items:
+            results.append(
+                {
+                    "template": "images.html",
+                    "url": item.get('link'),
+                    "thumbnail_src": item.get('thumb'),
+                    "img_src": item.get('originalUrl'),
+                    "title": html_to_text(item.get('title')),
+                    "source": item.get('source'),
+                    "resolution": f"{item.get('orgWidth')} x {item.get('orgHeight')}",
+                }
+            )
+
+    return results
+
+
+def parse_news(data):
+    results = EngineResults()
+    dom = html.fromstring(data)
+
+    for item in eval_xpath_list(
+        dom, "//div[contains(@class, 'sds-comps-base-layout') and contains(@class, 'sds-comps-full-layout')]"
+    ):
+        title = extract_text(eval_xpath(item, ".//span[contains(@class, 'sds-comps-text-type-headline1')]/text()"))
+
+        url = eval_xpath_getindex(item, ".//a[@href and @nocr='1']/@href", 0)
+
+        content = extract_text(eval_xpath(item, ".//span[contains(@class, 'sds-comps-text-type-body1')]"))
+
+        thumbnail = None
+        try:
+            thumbnail = eval_xpath_getindex(
+                item,
+                ".//div[contains(@class, 'sds-comps-image') and contains(@class, 'sds-rego-thumb-overlay')]//img[@src]/@src",
+                0,
+            )
+        except (ValueError, TypeError, SearxEngineXPathException):
+            pass
+
+        if title and content and url:
+            results.add(
+                MainResult(
+                    title=title,
+                    url=url,
+                    content=content,
+                    thumbnail=thumbnail,
+                )
+            )
+
+    return results
+
+
+def parse_videos(data):
+    results = []
+
+    dom = html.fromstring(data)
+
+    for item in eval_xpath_list(dom, "//li[contains(@class, 'video_item')]"):
+        thumbnail = None
+        try:
+            thumbnail = eval_xpath_getindex(item, ".//img[contains(@class, 'thumb')]/@src", 0)
+        except (ValueError, TypeError, SearxEngineXPathException):
+            pass
+
+        length = None
+        try:
+            length = parse_duration_string(extract_text(eval_xpath(item, ".//span[contains(@class, 'time')]")))
+        except (ValueError, TypeError):
+            pass
+
+        results.append(
+            {
+                "template": "videos.html",
+                "title": extract_text(eval_xpath(item, ".//a[contains(@class, 'info_title')]")),
+                "url": eval_xpath_getindex(item, ".//a[contains(@class, 'info_title')]/@href", 0),
+                "thumbnail": thumbnail,
+                'length': length,
+            }
+        )
+
+    return results

searx/engines/pinterest.py

@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Pinterest (images)
-"""
+"""Pinterest (images)"""
 
 from json import dumps
 
@@ -28,6 +27,11 @@ def request(query, params):
         'context': {},
     }
     params['url'] = f"{base_url}/resource/BaseSearchResource/get/?data={dumps(args)}"
+    params['headers'] = {
+        'X-Pinterest-AppState': 'active',
+        'X-Pinterest-Source-Url': '/ideas/',
+        'X-Pinterest-PWS-Handler': 'www/ideas.js',
+    }
 
     return params
 

searx/engines/selfhst.py

@@ -11,7 +11,7 @@ about = {
     "require_api_key": False,
     "results": 'JSON',
 }
-categories = ['images']
+categories = ['images', 'icons']
 
 
 icons_list_url = 'https://cdn.selfh.st/directory/icons.json'

searx/engines/svgrepo.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-"""Svgrepo (images)
+"""Svgrepo (icons)
 """
 
 from lxml import html
@@ -14,7 +14,7 @@ about = {
 }
 
 paging = True
-categories = ['images']
+categories = ['images', 'icons']
 base_url = "https://www.svgrepo.com"
 
 results_xpath = "//div[@class='style_nodeListing__7Nmro']/div"

searx/engines/uxwing.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""UXwing (images)"""
+
+from urllib.parse import quote_plus
+from lxml import html
+
+from searx.utils import eval_xpath, eval_xpath_list, extract_text
+
+about = {
+    "website": 'https://uxwing.com',
+    "wikidata_id": None,
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": 'HTML',
+}
+categories = ['images', 'icons']
+
+base_url = "https://uxwing.com"
+
+
+def request(query, params):
+    params['url'] = f"{base_url}/?s={quote_plus(query)}"
+    return params
+
+
+def response(resp):
+    results = []
+
+    doc = html.fromstring(resp.text)
+    for result in eval_xpath_list(doc, "//article[starts-with(@id, 'post')]"):
+        classes = extract_text(eval_xpath(result, "./@class")).split(" ")
+        tags = []
+        for css_class in classes:
+            for prefix in ("category", "tag"):
+                if css_class.startswith(prefix):
+                    tag = css_class.removeprefix(prefix)
+                    tags.append(tag.replace("-", " ").title())
+
+        results.append(
+            {
+                'template': 'images.html',
+                'url': extract_text(eval_xpath(result, "./a/@href")),
+                'img_src': extract_text(eval_xpath(result, ".//img/@src")),
+                'title': extract_text(eval_xpath(result, ".//img/@alt")),
+                'content': ', '.join(tags),
+            }
+        )
+
+    return results

searx/settings.yml

@@ -34,7 +34,7 @@ search:
   # Filter results. 0: None, 1: Moderate, 2: Strict
   safe_search: 0
   # Existing autocomplete backends: "360search", "baidu", "brave", "dbpedia", "duckduckgo", "google", "yandex",
-  # "mwmbl", "seznam", "sogou", "stract", "swisscows", "quark", "qwant", "wikipedia" -
+  # "mwmbl", "naver", "seznam", "sogou", "stract", "swisscows", "quark", "qwant", "wikipedia" -
   # leave blank to turn it off by default.
   autocomplete: ""
   # minimun characters to type before autocompleter starts
@@ -1053,7 +1053,6 @@ engines:
 
   - name: material icons
     engine: material_icons
-    categories: images
     shortcut: mi
     disabled: true
 
@@ -1236,11 +1235,13 @@ engines:
     shortcut: zlib
     categories: files
     timeout: 7.0
+    disabled: true
 
   - name: library of congress
     engine: loc
     shortcut: loc
     categories: images
+    disabled: true
 
   - name: libretranslate
     engine: libretranslate
@@ -1707,6 +1708,7 @@ engines:
     engine: qwant
     shortcut: qw
     categories: [general, web]
+    disabled: true
     additional_tests:
       rosebud: *test_rosebud
 
@@ -2358,25 +2360,31 @@ engines:
     disabled: true
 
   - name: naver
-    shortcut: nvr
     categories: [general, web]
-    engine: xpath
-    paging: true
-    search_url: https://search.naver.com/search.naver?where=webkr&sm=osp_hty&ie=UTF-8&query={query}&start={pageno}
-    url_xpath: //a[@class="link_tit"]/@href
-    title_xpath: //a[@class="link_tit"]
-    content_xpath: //div[@class="total_dsc_wrap"]/a
-    first_page_num: 1
-    page_size: 10
+    engine: naver
+    shortcut: nvr
+    disabled: true
+
+  - name: naver images
+    naver_category: images
+    categories: [images]
+    engine: naver
+    shortcut: nvri
+    disabled: true
+
+  - name: naver news
+    naver_category: news
+    categories: [news]
+    engine: naver
+    shortcut: nvrn
+    disabled: true
+
+  - name: naver videos
+    naver_category: videos
+    categories: [videos]
+    engine: naver
+    shortcut: nvrv
     disabled: true
-    about:
-      website: https://www.naver.com/
-      wikidata_id: Q485639
-      official_api_documentation: https://developers.naver.com/docs/nmt/examples/
-      use_official_api: false
-      require_api_key: false
-      results: HTML
-      language: ko
 
   - name: rubygems
     shortcut: rbg
@@ -2523,6 +2531,11 @@ engines:
     engine: tootfinder
     shortcut: toot
 
+  - name: uxwing
+    engine: uxwing
+    shortcut: ux
+    disabled: true
+
   - name: voidlinux
     engine: voidlinux
     shortcut: void

searx/utils.py

@@ -830,6 +830,11 @@ def js_variable_to_python(js_variable):
     s = _JS_DECIMAL_RE.sub(":0.", s)
     # replace the surogate character by colon
     s = s.replace(chr(1), ':')
+    # replace single-quote followed by comma with double-quote and comma
+    # {"a": "\"12\"',"b": "13"}
+    # becomes
+    # {"a": "\"12\"","b": "13"}
+    s = s.replace("',", "\",")
     # load the JSON and return the result
     return json.loads(s)