[fix] searchzee: circumvent botblocking

searx/engines/searchzee.py

@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""SearchZee is a small, indie project, web and news results pulled from
+independent search infrastructure."""
+
+import typing as t
+from urllib.parse import urlencode
+
+from searx.exceptions import SearxEngineAPIException
+from searx.extended_types import SXNG_Response
+from searx.network import get
+from searx.result_types import EngineResults
+from searx.utils import extr, html_to_text
+from searx.enginelib import EngineCache
+
+if t.TYPE_CHECKING:
+    from searx.search.processors import OnlineParams
+
+about = {
+    "website": "https://searchzee.com",
+    "official_api_documentation": None,
+    "use_official_api": False,
+    "require_api_key": False,
+    "results": "JSON",
+    "description": (
+        "SearchZee is a small, indie project, the web and news results"
+        " are pulled from an independent search infrastructure."
+    ),
+}
+categories: list[str] = None  # type: ignore[reportAssignmentType]
+
+paging = True
+
+SearchzeeCategType = t.Literal["web", "news"]
+searchzee_categ: SearchzeeCategType = None  # type: ignore[reportAssignmentType]
+
+
+CACHE: EngineCache
+"""Cache for storing the scraped API Token."""
+
+base_url = "https://searchzee.com"
+
+# only supports for news
+time_range_map = {"day": "pd", "week": "pw", "month": "pm", "year": "py"}
+
+
+def setup(engine_settings: dict[str, t.Any]) -> bool:
+    if searchzee_categ not in t.get_args(SearchzeeCategType):
+        raise ValueError("invalid category: %s" % searchzee_categ)
+
+    global CACHE  # pylint: disable=global-statement
+    CACHE = EngineCache(engine_settings["name"])  # type: ignore[reportAny]
+    return True
+
+
+def _obtain_api_token() -> str:
+    token: str | None = CACHE.get("token")  # type: ignore[reportAny]
+    if token:
+        return token
+
+    token_resp = get(
+        f"{base_url}/app.js",
+    )
+    if not token_resp.ok:
+        raise SearxEngineAPIException("failed to obtain api key")
+
+    token = extr(token_resp.text, "const SEARCHZEE_API_TOKEN = \"", "\";")
+    CACHE.set("token", token, expire=3600)
+
+    return token
+
+
+def request(query: str, params: "OnlineParams"):
+    params["headers"]["X-SearchZee-Token"] = _obtain_api_token()
+
+    args = {"q": query, "type": searchzee_categ, "offset": params["pageno"] - 1}
+    if params["time_range"]:
+        args["freshness"] = time_range_map[params["time_range"]]
+    params["url"] = f"{base_url}/api/search?{urlencode(args)}"
+
+
+def response(resp: "SXNG_Response") -> EngineResults:
+    res = EngineResults()
+
+    results: list[dict[str, str]] = resp.json()["results"]  # type: ignore[reportAny]
+
+    for result in results:
+        res.add(
+            res.types.MainResult(
+                url=result["url"],
+                title=html_to_text(result["title"]),
+                content=html_to_text(result["summary"]),
+                thumbnail=result.get("thumbnail") or "",
+            )
+        )
+
+    return res

searx/settings.yml

@@ -2989,44 +2989,17 @@ engines:
       results: HTML
 
   - name: searchzee
-    engine: json_engine
-    search_url: https://searchzee.com/api/search?q={query}&type=web&offset={pageno}
-    paging: true
-    first_page_num: 0
-    results_query: results
-    url_query: url
-    title_query: title
-    content_query: summary
-    content_html_to_text: true
+    engine: searchzee
     categories: general
+    searchzee_categ: web
     shortcut: sz
     disabled: true
     inactive: true
-    about:
-      website: https://searchzee.com
-      use_official_api: false
-      require_api_key: false
-      results: JSON
 
   - name: searchzee news
-    engine: json_engine
-    search_url: https://searchzee.com/api/search?q={query}&type=news&offset={pageno}{time_range}
-    paging: true
-    first_page_num: 0
-    time_range_support: true
-    time_range_url: "&freshness={time_range_val}"
-    time_range_map:
-      day: pd
-      week: pw
-      month: pm
-      year: py
-    results_query: results
-    url_query: url
-    title_query: title
-    content_query: summary
-    thumbnail_query: thumbnail
-    content_html_to_text: true
+    engine: searchzee
     categories: news
+    searchzee_categ: news
     shortcut: sznw
     disabled: true
     inactive: true