Merge f584ae31fa into 10d3af84b8

The query string send to DDG must not be qouted.

The query string was URL-qouted in #4011, but the URL-qouted query string result
in unexpected *URL decoded* and other garbish results as reported in #4019
and #4020.  To test compare the results of a query like::

    !ddg Häuser und Straßen :de
    !ddg Häuser und Straßen :all
    !ddg 房屋和街道 :all
    !ddg 房屋和街道 :zh

Closed:

- [#4019] https://github.com/searxng/searxng/issues/4019
- [#4020] https://github.com/searxng/searxng/issues/4020

Related:

- [#4011] https://github.com/searxng/searxng/pull/4011

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

AUTHORS.rst

@@ -174,3 +174,4 @@ features or generally made searx better:
 - @micsthepick
 - Daniel Kukula `<https://github.com/dkuku>`
 - Patrick Evans `https://github.com/holysoles`
+- Valentin Rieß `<https://github.com/v411e>`

docs/dev/engines/online/nixpkgs.rst

@@ -0,0 +1,8 @@
+.. _nixpkgs engine:
+
+==============
+Nixpkgs Search
+==============
+
+.. automodule:: searx.engines.nixpkgs
+   :members:

searx/engines/duckduckgo.py

@@ -6,7 +6,7 @@ DuckDuckGo Lite
 
 from typing import TYPE_CHECKING
 import re
-from urllib.parse import urlencode, quote_plus
+from urllib.parse import urlencode
 import json
 import babel
 import lxml.html
@@ -263,7 +263,7 @@ def request(query, params):
 
     params['url'] = url
     params['method'] = 'POST'
-    params['data']['q'] = quote_plus(query)
+    params['data']['q'] = query
 
     # The API is not documented, so we do some reverse engineering and emulate
     # what https://html.duckduckgo.com/html does when you press "next Page" link
@@ -381,7 +381,11 @@ def response(resp):
     zero_click_info_xpath = '//div[@id="zero_click_abstract"]'
     zero_click = extract_text(eval_xpath(doc, zero_click_info_xpath)).strip()
 
-    if zero_click and "Your IP address is" not in zero_click and "Your user agent:" not in zero_click:
+    if zero_click and (
+        "Your IP address is" not in zero_click
+        and "Your user agent:" not in zero_click
+        and "URL Decoded:" not in zero_click
+    ):
         current_query = resp.search_params["data"].get("q")
 
         results.append(

searx/engines/nixpkgs.py

@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""`Nixpkgs Search <https://search.nixos.org>`_ makes it easy to search the huge
+nixpkgs package index.  The search backend on search.nixos.org is an
+Elasticsearch instance.
+
+Example
+=======
+
+The following is an example configuration for an nixpkgs engine with
+authentication configured.
+
+Credentials are available here: https://github.com/NixOS/nixos-search/blob/main/frontend/src/index.js
+
+.. code:: yaml
+
+   - name: nixpkgs
+     shortcut: nix
+     engine: nixpkgs
+     base_url: https://search.nixos.org/backend
+     username:
+     password:
+     index: latest-42-nixos-unstable
+     channel: unstable
+
+"""
+
+from searx.exceptions import SearxEngineAPIException
+
+about = {
+    "website": "https://search.nixos.org",
+    "use_official_api": False,
+    "require_api_key": True,
+    "results": 'JSON',
+}
+categories = ["it", "packages"]
+
+base_url = "https://search.nixos.org/backend"
+username = ""
+password = ""
+index = "latest-42-nixos-unstable"
+channel = "unstable"
+search_url = f"{base_url}/{index}/_search"
+show_metadata = False
+categories = ["it", "packages"]
+
+
+def init():
+    if not (username and password):
+        raise ValueError("username and password need to be configured.")
+
+
+def request(query, params):
+    params["auth"] = (username, password)
+    params["url"] = search_url
+    params["data"] = _build_query(query)
+    params["headers"]["Content-Type"] = "application/json"
+
+    return params
+
+
+def response(resp):
+    results = []
+
+    resp_json = resp.json()
+    if "error" in resp_json:
+        raise SearxEngineAPIException(resp_json["error"])
+
+    for result in resp_json["hits"]["hits"]:
+        raw_result = {key: value if not key.startswith("_") else value for key, value in result["_source"].items()}
+        r = {
+            "template": "packages.html",
+            "title": raw_result["package_attr_name"],
+            "package_name": raw_result["package_pname"],
+            "version": raw_result["package_pversion"],
+            "content": raw_result["package_description"] or "",
+            "source_code_url": _position_to_github_url(raw_result["package_position"]),
+            "url": f"https://search.nixos.org/packages?channel={channel}&show={raw_result['package_attr_name']}",
+            "maintainer": ", ".join(raw_result["package_maintainers_set"]),
+            "tags": raw_result["package_programs"],
+            "license_name": ", ".join(raw_result["package_license_set"]),
+            "homepage": next(iter(raw_result["package_homepage"]), None),
+        }
+
+        if show_metadata:
+            r["metadata"] = {
+                "index": result["_index"],
+                "id": result["_id"],
+                "score": result["_score"],
+            }
+
+        results.append(r)
+
+    return results
+
+
+def _build_query(query: str):
+    return f"""
+      {{
+        "from": 0,
+        "size": 50,
+        "sort": [
+          {{
+            "_score": "desc",
+            "package_attr_name": "desc",
+            "package_pversion": "desc"
+          }}
+        ],
+        "query": {{
+          "bool": {{
+            "must": [
+              {{
+                "dis_max": {{
+                  "tie_breaker": 0.7,
+                  "queries": [
+                    {{
+                      "multi_match": {{
+                        "type": "cross_fields",
+                        "query": "{query}",
+                        "analyzer": "whitespace",
+                        "auto_generate_synonyms_phrase_query": false,
+                        "operator": "and",
+                        "_name": "multi_match_firefox",
+                        "fields": [
+                          "package_attr_name^9",
+                          "package_attr_name.*^5.3999999999999995",
+                          "package_programs^9",
+                          "package_programs.*^5.3999999999999995",
+                          "package_pname^6",
+                          "package_pname.*^3.5999999999999996",
+                          "package_description^1.3",
+                          "package_description.*^0.78",
+                          "package_longDescription^1",
+                          "package_longDescription.*^0.6",
+                          "flake_name^0.5",
+                          "flake_name.*^0.3"
+                        ]
+                      }}
+                    }},
+                    {{
+                      "wildcard": {{
+                        "package_attr_name": {{
+                          "value": "*{query}*",
+                          "case_insensitive": true
+                        }}
+                      }}
+                    }}
+                  ]
+                }}
+              }}
+            ]
+          }}
+        }}
+      }}
+    """
+
+
+def _position_to_github_url(package_position: str):
+    path, line = package_position.split(":")
+    return f"https://github.com/NixOS/nixpkgs/blob/master/{path}#L{line}"
+
+
+def _get_codelines(package_name: str):
+    code = f"nix-shell -p {package_name}"
+    return [(0, code)]

searx/settings.yml

@@ -2406,6 +2406,16 @@ engines:
     shortcut: pgo
     disabled: true
 
+# NixOS Package Search
+# - name: nixpkgs
+#   shortcut: nix
+#   engine: nixpkgs
+#   base_url: https://search.nixos.org/backend
+#   username:
+#   password:
+#   index: latest-42-nixos-unstable
+#   channel: unstable
+
 # Doku engine lets you access to any Doku wiki instance:
 # A public one or a privete/corporate one.
 #  - name: ubuntuwiki