[chore] add timediff field for sogou_videos

Co-authored-by: Bnyro <bnyro@tutanota.com>

docs/admin/settings/settings_search.rst

@@ -33,6 +33,7 @@
 ``autocomplete``:
   Existing autocomplete backends, leave blank to turn it off.
 
+  - ``360search``
   - ``baidu``
   - ``brave``
   - ``dbpedia``
@@ -41,6 +42,7 @@
   - ``mwmbl``
   - ``qwant``
   - ``seznam``
+  - ``sogou``
   - ``stract``
   - ``swisscows``
   - ``wikipedia``

searx/autocomplete.py

@@ -20,6 +20,7 @@ from searx.engines import (
 )
 from searx.network import get as http_get, post as http_post
 from searx.exceptions import SearxEngineResponseException
+from searx.utils import extr
 
 
 def update_kwargs(**kwargs):
@@ -148,6 +149,21 @@ def mwmbl(query, _lang):
     return [result for result in results if not result.startswith("go: ") and not result.startswith("search: ")]
 
 
+def qihu360search(query, _lang):
+    # 360Search search autocompleter
+    url = f"https://sug.so.360.cn/suggest?{urlencode({'format': 'json', 'word': query})}"
+    response = get(url)
+
+    results = []
+
+    if response.ok:
+        data = response.json()
+        if 'result' in data:
+            for item in data['result']:
+                results.append(item['word'])
+    return results
+
+
 def seznam(query, _lang):
     # seznam search autocompleter
     url = 'https://suggest.seznam.cz/fulltext/cs?{query}'
@@ -171,6 +187,23 @@ def seznam(query, _lang):
     ]
 
 
+def sogou(query, _lang):
+    # Sogou search autocompleter
+    base_url = "https://sor.html5.qq.com/api/getsug?"
+    response = get(base_url + urlencode({'m': 'searxng', 'key': query}))
+
+    if response.ok:
+        raw_json = extr(response.text, "[", "]", default="")
+
+        try:
+            data = json.loads(f"[{raw_json}]]")
+            return data[1]
+        except json.JSONDecodeError:
+            return []
+
+    return []
+
+
 def stract(query, _lang):
     # stract autocompleter (beta)
     url = f"https://stract.com/beta/api/autosuggest?q={quote_plus(query)}"
@@ -246,6 +279,7 @@ def yandex(query, _lang):
 
 
 backends = {
+    '360search': qihu360search,
     'baidu': baidu,
     'brave': brave,
     'dbpedia': dbpedia,
@@ -254,6 +288,7 @@ backends = {
     'mwmbl': mwmbl,
     'qwant': qwant,
     'seznam': seznam,
+    'sogou': sogou,
     'stract': stract,
     'swisscows': swisscows,
     'wikipedia': wikipedia,

searx/engines/360search.py

@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=invalid-name
+"""360Search search engine for searxng"""
+
+from urllib.parse import urlencode
+from lxml import html
+
+from searx.utils import extract_text
+
+# Metadata
+about = {
+    "website": "https://www.so.com/",
+    "wikidata_id": "Q10846064",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+}
+
+# Engine Configuration
+categories = ["general"]
+paging = True
+time_range_support = True
+
+time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
+
+# Base URL
+base_url = "https://www.so.com"
+
+
+def request(query, params):
+    query_params = {
+        "pn": params["pageno"],
+        "q": query,
+    }
+
+    if time_range_dict.get(params['time_range']):
+        query_params["adv_t"] = time_range_dict.get(params['time_range'])
+
+    params["url"] = f"{base_url}/s?{urlencode(query_params)}"
+    return params
+
+
+def response(resp):
+    dom = html.fromstring(resp.text)
+    results = []
+
+    for item in dom.xpath('//li[contains(@class, "res-list")]'):
+        title = extract_text(item.xpath('.//h3[contains(@class, "res-title")]/a'))
+
+        url = extract_text(item.xpath('.//h3[contains(@class, "res-title")]/a/@data-mdurl'))
+        if not url:
+            url = extract_text(item.xpath('.//h3[contains(@class, "res-title")]/a/@href'))
+
+        content = extract_text(item.xpath('.//p[@class="res-desc"]'))
+        if not content:
+            content = extract_text(item.xpath('.//span[@class="res-list-summary"]'))
+
+        if title and url:
+            results.append(
+                {
+                    "title": title,
+                    "url": url,
+                    "content": content,
+                }
+            )
+
+    return results

searx/engines/360search_videos.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=invalid-name
+"""360Search-Videos: A search engine for retrieving videos from 360Search."""
+
+from urllib.parse import urlencode
+from datetime import datetime
+
+from searx.exceptions import SearxEngineAPIException
+from searx.utils import html_to_text
+
+about = {
+    "website": "https://tv.360kan.com/",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "JSON",
+}
+
+paging = True
+results_per_page = 10
+categories = ["videos"]
+
+base_url = "https://tv.360kan.com"
+
+
+def request(query, params):
+    query_params = {"count": 10, "q": query, "start": params["pageno"] * 10}
+
+    params["url"] = f"{base_url}/v1/video/list?{urlencode(query_params)}"
+    return params
+
+
+def response(resp):
+    try:
+        data = resp.json()
+    except Exception as e:
+        raise SearxEngineAPIException(f"Invalid response: {e}") from e
+    results = []
+
+    if "data" not in data or "result" not in data["data"]:
+        raise SearxEngineAPIException("Invalid response")
+
+    for entry in data["data"]["result"]:
+        if not entry.get("title") or not entry.get("play_url"):
+            continue
+
+        published_date = None
+        if entry.get("publish_time"):
+            try:
+                published_date = datetime.fromtimestamp(int(entry["publish_time"]))
+            except (ValueError, TypeError):
+                published_date = None
+
+        results.append(
+            {
+                'url': entry["play_url"],
+                'title': html_to_text(entry["title"]),
+                'content': html_to_text(entry["description"]),
+                'template': 'videos.html',
+                'publishedDate': published_date,
+                'thumbnail': entry["cover_img"],
+            }
+        )
+
+    return results

searx/engines/sogou.py

@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Sogou search engine for searxng"""
+
+from urllib.parse import urlencode
+from lxml import html
+
+from searx.utils import extract_text
+
+# Metadata
+about = {
+    "website": "https://www.sogou.com/",
+    "wikidata_id": "Q7554565",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+}
+
+# Engine Configuration
+categories = ["general"]
+paging = True
+time_range_support = True
+
+time_range_dict = {'day': 'inttime_day', 'week': 'inttime_week', 'month': 'inttime_month', 'year': 'inttime_year'}
+
+# Base URL
+base_url = "https://www.sogou.com"
+
+
+def request(query, params):
+    query_params = {
+        "query": query,
+        "page": params["pageno"],
+    }
+
+    if time_range_dict.get(params['time_range']):
+        query_params["s_from"] = time_range_dict.get(params['time_range'])
+        query_params["tsn"] = 1
+
+    params["url"] = f"{base_url}/web?{urlencode(query_params)}"
+    return params
+
+
+def response(resp):
+    dom = html.fromstring(resp.text)
+    results = []
+
+    for item in dom.xpath('//div[contains(@class, "vrwrap")]'):
+        title = extract_text(item.xpath('.//h3[contains(@class, "vr-title")]/a'))
+        url = extract_text(item.xpath('.//h3[contains(@class, "vr-title")]/a/@href'))
+
+        if url.startswith("/link?url="):
+            url = f"{base_url}{url}"
+
+        content = extract_text(item.xpath('.//div[contains(@class, "text-layout")]//p[contains(@class, "star-wiki")]'))
+        if not content:
+            content = extract_text(item.xpath('.//div[contains(@class, "fz-mid space-txt")]'))
+
+        if title and url:
+            results.append(
+                {
+                    "title": title,
+                    "url": url,
+                    "content": content,
+                }
+            )
+
+    return results

searx/engines/sogou_images.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Sogou-Images: A search engine for retrieving images from Sogou."""
+
+import json
+import re
+from urllib.parse import urlencode
+
+# about
+about = {
+    "website": "https://pic.sogou.com/",
+    "wikidata_id": "Q7554565",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+}
+
+# engine dependent config
+categories = ["images"]
+paging = True
+
+base_url = "https://pic.sogou.com"
+
+
+def request(query, params):
+    query_params = {
+        "query": query,
+        "start": (params["pageno"] - 1) * 48,
+    }
+
+    params["url"] = f"{base_url}/pics?{urlencode(query_params)}"
+    return params
+
+
+def response(resp):
+    results = []
+    match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.*?});', resp.text, re.S)
+    if not match:
+        return results
+
+    data = json.loads(match.group(1))
+    if "searchList" in data and "searchList" in data["searchList"]:
+        for item in data["searchList"]["searchList"]:
+            results.append(
+                {
+                    "template": "images.html",
+                    "url": item.get("url", ""),
+                    "thumbnail_src": item.get("picUrl", ""),
+                    "img_src": item.get("picUrl", ""),
+                    "content": item.get("content_major", ""),
+                    "title": item.get("title", ""),
+                    "source": item.get("ch_site_name", ""),
+                }
+            )
+
+    return results

searx/engines/sogou_videos.py

@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Sogou-Videos: A search engine for retrieving videos from Sogou."""
+
+from urllib.parse import urlencode
+from datetime import datetime, timedelta
+
+from searx.exceptions import SearxEngineAPIException
+
+about = {
+    "website": "https://v.sogou.com/",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "JSON",
+}
+
+categories = ["videos"]
+paging = True
+results_per_page = 10
+
+# Base URL
+base_url = "https://v.sogou.com"
+
+
+def request(query, params):
+    query_params = {
+        "page": params["pageno"],
+        "pagesize": 10,
+        "query": query,
+    }
+
+    params["url"] = f"{base_url}/api/video/shortVideoV2?{urlencode(query_params)}"
+    return params
+
+
+def response(resp):
+    try:
+        data = resp.json()
+    except Exception as e:
+        raise SearxEngineAPIException(f"Invalid response: {e}") from e
+    results = []
+
+    if not data.get("data", {}).get("list"):
+        raise SearxEngineAPIException("Invalid response")
+
+    for entry in data["data"]["list"]:
+        if not entry.get("titleEsc") or not entry.get("url"):
+            continue
+
+        video_url = entry.get("url")
+        if video_url.startswith("/vc/np"):
+            video_url = f"{base_url}{video_url}"
+
+        published_date = None
+        if entry.get("date") and entry.get("duration"):
+            try:
+                published_date = datetime.strptime(entry['date'], "%Y-%m-%d")
+            except (ValueError, TypeError):
+                published_date = None
+
+        length = None
+        if entry.get("date") and entry.get("duration"):
+            try:
+                timediff = datetime.strptime(entry['duration'], "%M:%S")
+                length = timedelta(minutes=timediff.minute, seconds=timediff.second)
+            except (ValueError, TypeError):
+                length = None
+
+        results.append(
+            {
+                'url': video_url,
+                'title': entry["titleEsc"],
+                'content': entry['site'],
+                'length': length,
+                'template': 'videos.html',
+                'publishedDate': published_date,
+                'thumbnail': entry["picurl"],
+            }
+        )
+
+    return results

searx/engines/sogou_wechat.py

@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Sogou-WeChat search engine for retrieving WeChat Article from Sogou"""
+
+from urllib.parse import urlencode
+from datetime import datetime
+import re
+from lxml import html
+
+from searx.utils import extract_text
+
+# Metadata
+about = {
+    "website": "https://weixin.sogou.com/",
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "HTML",
+}
+
+# Engine Configuration
+categories = ["news"]
+paging = True
+
+# Base URL
+base_url = "https://weixin.sogou.com"
+
+
+def request(query, params):
+    query_params = {
+        "query": query,
+        "page": params["pageno"],
+        "type": 2,
+    }
+
+    params["url"] = f"{base_url}/weixin?{urlencode(query_params)}"
+    return params
+
+
+def response(resp):
+    dom = html.fromstring(resp.text)
+    results = []
+
+    for item in dom.xpath('//li[contains(@id, "sogou_vr_")]'):
+        title = extract_text(item.xpath('.//h3/a'))
+        url = extract_text(item.xpath('.//h3/a/@href'))
+
+        if url.startswith("/link?url="):
+            url = f"{base_url}{url}"
+
+        content = extract_text(item.xpath('.//p[@class="txt-info"]'))
+        if not content:
+            content = extract_text(item.xpath('.//p[contains(@class, "txt-info")]'))
+
+        thumbnail = extract_text(item.xpath('.//div[@class="img-box"]/a/img/@src'))
+        if thumbnail and thumbnail.startswith("//"):
+            thumbnail = f"https:{thumbnail}"
+
+        published_date = None
+        timestamp = extract_text(item.xpath('.//script[contains(text(), "timeConvert")]'))
+        if timestamp:
+            match = re.search(r"timeConvert\('(\d+)'\)", timestamp)
+            if match:
+                published_date = datetime.fromtimestamp(int(match.group(1)))
+
+        if title and url:
+            results.append(
+                {
+                    "title": title,
+                    "url": url,
+                    "content": content,
+                    'thumbnail': thumbnail,
+                    "publishedDate": published_date,
+                }
+            )
+
+    return results

searx/settings.yml

@@ -33,8 +33,8 @@ brand:
 search:
   # Filter results. 0: None, 1: Moderate, 2: Strict
   safe_search: 0
-  # Existing autocomplete backends: "baidu", "brave", "dbpedia", "duckduckgo", "google", "yandex",
-  # "mwmbl", "seznam", "stract", "swisscows", "qwant", "wikipedia" -
+  # Existing autocomplete backends: "360search", "baidu", "brave", "dbpedia", "duckduckgo", "google", "yandex",
+  # "mwmbl", "seznam", "sogou", "stract", "swisscows", "qwant", "wikipedia" -
   # leave blank to turn it off by default.
   autocomplete: ""
   # minimun characters to type before autocompleter starts
@@ -337,6 +337,16 @@ categories_as_tabs:
   social media:
 
 engines:
+  - name: 360search
+    engine: 360search
+    shortcut: 360so
+    disabled: true
+
+  - name: 360search videos
+    engine: 360search_videos
+    shortcut: 360sov
+    disabled: true
+
   - name: 9gag
     engine: 9gag
     shortcut: 9g
@@ -1701,6 +1711,26 @@ engines:
     engine: sepiasearch
     shortcut: sep
 
+  - name: sogou
+    engine: sogou
+    shortcut: sogou
+    disabled: true
+
+  - name: sogou images
+    engine: sogou_images
+    shortcut: sogoui
+    disabled: true
+
+  - name: sogou videos
+    engine: sogou_videos
+    shortcut: sogouv
+    disabled: true
+
+  - name: sogou wechat
+    engine: sogou_wechat
+    shortcut: sogouw
+    disabled: true
+
   - name: soundcloud
     engine: soundcloud
     shortcut: sc