[build] /static

Closes: https://github.com/searxng/searxng/issues/4461
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

client/simple/package-lock.json

@@ -11,11 +11,11 @@
         "autocomplete-js": "^2.7.1"
       },
       "devDependencies": {
-        "@eslint/js": "^9.21.0",
+        "@eslint/js": "^9.22.0",
         "copy-webpack-plugin": "^13.0.0",
         "css-loader": "^7.1.2",
         "edge.js": "^6.2.1",
-        "eslint": "^9.21.0",
+        "eslint": "^9.22.0",
         "filemanager-webpack-plugin": "^8.0.0",
         "globals": "^16.0.0",
         "ionicons": "^7.4.0",
@@ -30,7 +30,7 @@
         "stylelint-config-standard-less": "^3.0.1",
         "stylelint-prettier": "^5.0.3",
         "svgo": "^3.3.2",
-        "vite": "^6.2.1",
+        "vite": "^6.2.2",
         "vite-plugin-static-copy": "^2.3.0",
         "vite-plugin-stylelint": "^6.0.0",
         "webpack": "^5.97.1",
@@ -666,6 +666,16 @@
         "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
       }
     },
+    "node_modules/@eslint/config-helpers": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.1.0.tgz",
+      "integrity": "sha512-kLrdPDJE1ckPo94kmPPf9Hfd0DU0Jw6oKYrhe+pwSC0iTUInmTa+w6fw8sGgcfkFJGNdWOUeOaDM4quW4a7OkA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "engines": {
+        "node": "^18.18.0 || ^20.9.0 || >=21.1.0"
+      }
+    },
     "node_modules/@eslint/core": {
       "version": "0.12.0",
       "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.12.0.tgz",
@@ -717,9 +727,9 @@
       }
     },
     "node_modules/@eslint/js": {
-      "version": "9.21.0",
+      "version": "9.22.0",
-      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.22.0.tgz",
-      "integrity": "sha512-BqStZ3HX8Yz6LvsF5ByXYrtigrV5AXADWLAGc7PH/1SxOb7/FIYYMszZZWiUou/GB9P2lXWk2SV4d+Z8h0nknw==",
+      "integrity": "sha512-vLFajx9o8d1/oL2ZkpMYbkLv8nDB6yaIwFNt7nI4+I80U/z03SxmfOMsLbvWr3p7C+Wnoh//aOu2pQW8cS0HCQ==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -3412,18 +3422,19 @@
       }
     },
     "node_modules/eslint": {
-      "version": "9.21.0",
+      "version": "9.22.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.22.0.tgz",
-      "integrity": "sha512-KjeihdFqTPhOMXTt7StsDxriV4n66ueuF/jfPNC3j/lduHwr/ijDwJMsF+wyMJethgiKi5wniIE243vi07d3pg==",
+      "integrity": "sha512-9V/QURhsRN40xuHXWjV64yvrzMjcz7ZyNoF2jJFmy9j/SLk0u1OLSZgXi28MrXjymnjEGSR80WCdab3RGMDveQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.2.0",
         "@eslint-community/regexpp": "^4.12.1",
         "@eslint/config-array": "^0.19.2",
+        "@eslint/config-helpers": "^0.1.0",
         "@eslint/core": "^0.12.0",
         "@eslint/eslintrc": "^3.3.0",
-        "@eslint/js": "9.21.0",
+        "@eslint/js": "9.22.0",
         "@eslint/plugin-kit": "^0.2.7",
         "@humanfs/node": "^0.16.6",
         "@humanwhocodes/module-importer": "^1.0.1",
@@ -3435,7 +3446,7 @@
         "cross-spawn": "^7.0.6",
         "debug": "^4.3.2",
         "escape-string-regexp": "^4.0.0",
-        "eslint-scope": "^8.2.0",
+        "eslint-scope": "^8.3.0",
         "eslint-visitor-keys": "^4.2.0",
         "espree": "^10.3.0",
         "esquery": "^1.5.0",
@@ -3472,9 +3483,9 @@
       }
     },
     "node_modules/eslint-scope": {
-      "version": "8.2.0",
+      "version": "8.3.0",
-      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.3.0.tgz",
-      "integrity": "sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A==",
+      "integrity": "sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==",
       "dev": true,
       "license": "BSD-2-Clause",
       "dependencies": {
@@ -6755,9 +6766,9 @@
       "license": "MIT"
     },
     "node_modules/vite": {
-      "version": "6.2.1",
+      "version": "6.2.2",
-      "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.2.tgz",
-      "integrity": "sha512-n2GnqDb6XPhlt9B8olZPrgMD/es/Nd1RdChF6CBD/fHW6pUyUTt2sQW2fPRX5GiD9XEa6+8A6A4f2vT6pSsE7Q==",
+      "integrity": "sha512-yW7PeMM+LkDzc7CgJuRLMW2Jz0FxMOsVJ8Lv3gpgW9WLcb9cTW+121UEr1hvmfR7w3SegR5ItvYyzVz1vxNJgQ==",
       "dev": true,
       "license": "MIT",
       "dependencies": {

client/simple/package.json

@@ -9,11 +9,11 @@
     "icons.html": "node theme_icons.js"
   },
   "devDependencies": {
-    "@eslint/js": "^9.21.0",
+    "@eslint/js": "^9.22.0",
     "copy-webpack-plugin": "^13.0.0",
     "css-loader": "^7.1.2",
     "edge.js": "^6.2.1",
-    "eslint": "^9.21.0",
+    "eslint": "^9.22.0",
     "filemanager-webpack-plugin": "^8.0.0",
     "globals": "^16.0.0",
     "ionicons": "^7.4.0",
@@ -28,7 +28,7 @@
     "stylelint-config-standard-less": "^3.0.1",
     "stylelint-prettier": "^5.0.3",
     "svgo": "^3.3.2",
-    "vite": "^6.2.1",
+    "vite": "^6.2.2",
     "vite-plugin-static-copy": "^2.3.0",
     "vite-plugin-stylelint": "^6.0.0",
     "webpack": "^5.97.1",

client/simple/src/less/definitions.less

@@ -57,6 +57,10 @@
   /// Answer Colors
   --color-answer-font: #444; // same as --color-base-font
   --color-answer-background: #fff;
+  // colors of the KeyValue result class
+  --color-result-keyvalue-col-table: #fdfbff;
+  --color-result-keyvalue-odd: #fdfbff;
+  --color-result-keyvalue-even: #fff;
   /// Results Colors
   --color-result-background: #fff;
   --color-result-border: #ddd;
@@ -180,6 +184,10 @@
   /// Answer Colors
   --color-answer-font: #bbb; // same as --color-base-font
   --color-answer-background: #26292f;
+  // colors of the KeyValue result class
+  --color-result-keyvalue-col-table: #1e1e22;
+  --color-result-keyvalue-odd: #1e1e22;
+  --color-result-keyvalue-even: #26292f;
   /// Results Colors
   --color-result-background: #26292f;
   --color-result-border: #333;

client/simple/src/less/result_types/keyvalue.less

@@ -0,0 +1,35 @@
+/*
+  Layout of the KeyValue result class
+*/
+#main_results .result-keyvalue {
+  caption {
+    padding: 0.8rem 0.5rem;
+    font-style: italic;
+    caption-side: bottom;
+    background-color: var(--color-result-keyvalue-table);
+  }
+
+  .col-key {
+    width: 25%;
+  }
+
+  table {
+    word-break: break-word;
+    table-layout: fixed;
+    width: 100%;
+    background-color: var(--color-result-keyvalue-table);
+  }
+
+  tr.odd {
+    background-color: var(--color-result-keyvalue-odd);
+  }
+
+  tr.even {
+    background-color: var(--color-result-keyvalue-even);
+  }
+
+  th,
+  td {
+    padding: 0.3rem 0.5rem;
+  }
+}

client/simple/src/less/style.less

@@ -80,6 +80,12 @@ body {
   margin: 0;
 }
 
+@supports (height: 100dvh) {
+  body {
+    height: 100dvh;
+  }
+}
+
 main {
   width: 100%;
   margin-bottom: 2rem;
@@ -1158,3 +1164,6 @@ summary.title {
 pre code {
   white-space: pre-wrap;
 }
+
+// import layouts of the Result types
+@import "result_types/keyvalue.less";

docs/dev/result_types/answer.rst

@@ -4,4 +4,7 @@
 Answer Results
 ==============
 
+The :ref:`area answer results` is an area in which additional information can
+be displayed.
+
 .. automodule:: searx.result_types.answer

docs/dev/result_types/correction.rst

@@ -9,7 +9,7 @@ Correction Results
    There is still no typing for these result items. The templates can be used as
    orientation until the final typing is complete.
 
-The corrections area shows the user alternative search terms.
+The :ref:`area corrections results` shows the user alternative search terms.
 
 A result of this type is a very simple dictionary with only one key/value pair
 

docs/dev/result_types/index.rst

@@ -18,22 +18,32 @@ The **sources** can be:
 The sources provide the results, which are displayed in different **areas**
 depending on the type of result. The areas are:
 
-main results:
+.. _area main results:
+
+:ref:`area main results <main search results>`
   It is the main area in which -- as is typical for search engines -- the
   results that a search engine has found for the search term are displayed.
 
-answers:
+.. _area answer results:
+
+:ref:`area answers <result_types.answer>`
   This area displays short answers that could be found for the search term.
 
-info box:
+.. _area info box:
+
+:ref:`area info box <result_types.infobox>`
   An area in which additional information can be displayed, e.g. excerpts from
   wikipedia or other sources such as maps.
 
-suggestions:
+.. _area suggestions results:
+
+:ref:`area suggestions <result_types.suggestion>`
   Suggestions for alternative search terms can be found in this area.  These can
   be clicked on and a search is carried out with these search terms.
 
-corrections:
+.. _area corrections results:
+
+:ref:`area corrections <result_types.corrections>`
   Results in this area are like the suggestion of alternative search terms,
   which usually result from spelling corrections
 

docs/dev/result_types/infobox.rst

@@ -9,7 +9,7 @@ Infobox Results
    There is still no typing for these result items. The templates can be used as
    orientation until the final typing is complete.
 
-The infobox is an area where addtional infos shown to the user.
+The :ref:`area info box` is an area where addtional infos shown to the user.
 
 Fields used in the :origin:`infobox.html
 <searx/templates/simple/elements/infobox.html>`:
@@ -57,4 +57,3 @@ relatedTopics: :py:class:`List <list>`\ [\ :py:class:`dict`\ ]
     key/value pair:
 
     - suggestion: :py:class:`str`: suggested search term (mandatory)
-

docs/dev/result_types/main/keyvalue.rst

@@ -0,0 +1,7 @@
+.. _result_types.keyvalue:
+
+=================
+Key-Value Results
+=================
+
+.. automodule:: searx.result_types.keyvalue

docs/dev/result_types/main/mainresult.rst

@@ -0,0 +1,4 @@
+.. _result_types.mainresult:
+
+.. autoclass:: searx.result_types._base.MainResult
+   :members:

docs/dev/result_types/main_result.rst

@@ -1,11 +1,26 @@
-============
+.. _main search results:
-Main Results
-============
 
-There is still no typing for the items in the :ref:`main result list`.  The
+===================
-templates can be used as orientation until the final typing is complete.
+Main Search Results
+===================
 
-- :ref:`template default`
+In the :ref:`area main results` the results that a search engine has found for
+the search term are displayed.
+
+There is still no typing for all items in the :ref:`main result list`.  The
+following types have been implemented so far ..
+
+.. toctree::
+   :maxdepth: 2
+
+   main/mainresult
+   main/keyvalue
+
+The :ref:`LegacyResult <LegacyResult>` is used internally for the results that
+have not yet been typed.  The templates can be used as orientation until the
+final typing is complete.
+
+- :ref:`template default` / :py:obj:`Result`
 - :ref:`template images`
 - :ref:`template videos`
 - :ref:`template torrent`

docs/dev/result_types/suggestion.rst

@@ -9,7 +9,7 @@ Suggestion Results
    There is still no typing for these result items. The templates can be used as
    orientation until the final typing is complete.
 
-The suggestions area shows the user alternative search terms.
+The :ref:`area suggestions results` shows the user alternative search terms.
 
 A result of this type is a very simple dictionary with only one key/value pair
 

requirements-dev.txt

@@ -15,8 +15,8 @@ sphinx-autobuild==2024.10.3
 sphinx-notfound-page==1.1.0
 myst-parser==3.0.1
 linuxdoc==20240924
-aiounittest==1.4.3
+aiounittest==1.5.0
-yamllint==1.35.1
+yamllint==1.36.0
 wlc==1.15
 coloredlogs==15.0.1
 docutils>=0.21.2

searx/botdetection/_helpers.py

@@ -8,6 +8,7 @@ from ipaddress import (
     IPv4Address,
     IPv6Address,
     ip_network,
+    ip_address,
 )
 import flask
 import werkzeug
@@ -125,6 +126,9 @@ def get_real_ip(request: SXNG_Request) -> str:
     if real_ip and remote_addr and real_ip != remote_addr:
         logger.warning("IP from WSGI environment (%s) is not equal to IP from X-Real-IP (%s)", remote_addr, real_ip)
 
-    request_ip = forwarded_for or real_ip or remote_addr or '0.0.0.0'
+    request_ip = ip_address(forwarded_for or real_ip or remote_addr or '0.0.0.0')
+    if request_ip.version == 6 and request_ip.ipv4_mapped:
+        request_ip = request_ip.ipv4_mapped
+
     # logger.debug("get_real_ip() -> %s", request_ip)
-    return request_ip
+    return str(request_ip)

searx/enginelib/__init__.py

@@ -142,3 +142,6 @@ class Engine:  # pylint: disable=too-few-public-methods
     tokens: List[str]
     """A list of secret tokens to make this engine *private*, more details see
     :ref:`private engines`."""
+
+    weight: int
+    """Weighting of the results of this engine (:ref:`weight <settings engines>`)."""

searx/engines/command.py

@@ -81,6 +81,7 @@ from subprocess import Popen, PIPE
 from threading import Thread
 
 from searx import logger
+from searx.result_types import EngineResults
 
 
 engine_type = 'offline'
@@ -93,7 +94,6 @@ query_enum = []
 environment_variables = {}
 working_dir = realpath('.')
 result_separator = '\n'
-result_template = 'key-value.html'
 timeout = 4.0
 
 _command_logger = logger.getChild('command')
@@ -126,17 +126,17 @@ def init(engine_settings):
         environment_variables = engine_settings['environment_variables']
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
+    res = EngineResults()
     cmd = _get_command_to_run(query)
     if not cmd:
-        return []
+        return res
 
-    results = []
+    reader_thread = Thread(target=_get_results_from_process, args=(res, cmd, params['pageno']))
-    reader_thread = Thread(target=_get_results_from_process, args=(results, cmd, params['pageno']))
     reader_thread.start()
     reader_thread.join(timeout=timeout)
 
-    return results
+    return res
 
 
 def _get_command_to_run(query):
@@ -153,7 +153,7 @@ def _get_command_to_run(query):
     return cmd
 
 
-def _get_results_from_process(results, cmd, pageno):
+def _get_results_from_process(res: EngineResults, cmd, pageno):
     leftover = ''
     count = 0
     start, end = __get_results_limits(pageno)
@@ -173,12 +173,11 @@ def _get_results_from_process(results, cmd, pageno):
                     continue
 
                 if start <= count and count <= end:  # pylint: disable=chained-comparison
-                    result['template'] = result_template
+                    res.add(res.types.KeyValue(kvmap=result))
-                    results.append(result)
 
                 count += 1
                 if end < count:
-                    return results
+                    return res
 
             line = process.stdout.readline()
 

searx/engines/demo_offline.py

@@ -13,6 +13,7 @@ close to the implementation, its just a simple example.  To get in use of this
 """
 
 import json
+
 from searx.result_types import EngineResults
 
 engine_type = 'offline'
@@ -29,7 +30,7 @@ about = {
 }
 
 # if there is a need for globals, use a leading underline
-_my_offline_engine = None
+_my_offline_engine: str = ""
 
 
 def init(engine_settings=None):
@@ -50,24 +51,28 @@ def init(engine_settings=None):
 
 
 def search(query, request_params) -> EngineResults:
-    """Query (offline) engine and return results.  Assemble the list of results from
+    """Query (offline) engine and return results.  Assemble the list of results
-    your local engine.  In this demo engine we ignore the 'query' term, usual
+    from your local engine.  In this demo engine we ignore the 'query' term,
-    you would pass the 'query' term to your local engine to filter out the
+    usual you would pass the 'query' term to your local engine to filter out the
     results.
-
     """
     res = EngineResults()
 
-    result_list = json.loads(_my_offline_engine)
+    count = 0
-
+    for row in json.loads(_my_offline_engine):
-    for row in result_list:
+        count += 1
-        entry = {
+        kvmap = {
             'query': query,
             'language': request_params['searxng_locale'],
             'value': row.get("value"),
-            # choose a result template or comment out to use the *default*
-            'template': 'key-value.html',
         }
-        res.append(entry)
+        res.add(
-
+            res.types.KeyValue(
+                caption=f"Demo Offline Engine Result #{count}",
+                key_title="Name",
+                value_title="Value",
+                kvmap=kvmap,
+            )
+        )
+    res.add(res.types.LegacyResult(number_of_results=count))
     return res

searx/engines/elasticsearch.py

@@ -43,6 +43,8 @@ authentication configured to read from ``my-index`` index.
 
 from json import loads, dumps
 from searx.exceptions import SearxEngineAPIException
+from searx.result_types import EngineResults
+from searx.extended_types import SXNG_Response
 
 
 base_url = 'http://localhost:9200'
@@ -145,23 +147,20 @@ def _custom_query(query):
     return custom_query
 
 
-def response(resp):
+def response(resp: SXNG_Response) -> EngineResults:
-    results = []
+    res = EngineResults()
 
     resp_json = loads(resp.text)
     if 'error' in resp_json:
-        raise SearxEngineAPIException(resp_json['error'])
+        raise SearxEngineAPIException(resp_json["error"])
-
-    for result in resp_json['hits']['hits']:
-        r = {key: str(value) if not key.startswith('_') else value for key, value in result['_source'].items()}
-        r['template'] = 'key-value.html'
 
+    for result in resp_json["hits"]["hits"]:
+        kvmap = {key: str(value) if not key.startswith("_") else value for key, value in result["_source"].items()}
         if show_metadata:
-            r['metadata'] = {'index': result['_index'], 'id': result['_id'], 'score': result['_score']}
+            kvmap["metadata"] = {"index": result["_index"], "id": result["_id"], "score": result["_score"]}
+        res.add(res.types.KeyValue(kvmap=kvmap))
 
-        results.append(r)
+    return res
-
-    return results
 
 
 _available_query_types = {

searx/engines/mariadb_server.py

@@ -35,6 +35,8 @@ except ImportError:
     # the engine
     pass
 
+from searx.result_types import EngineResults
+
 if TYPE_CHECKING:
     import logging
 
@@ -63,7 +65,6 @@ query_str = ""
 
 limit = 10
 paging = True
-result_template = 'key-value.html'
 _connection = None
 
 
@@ -79,17 +80,16 @@ def init(engine_settings):
     _connection = mariadb.connect(database=database, user=username, password=password, host=host, port=port)
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
     query_params = {'query': query}
     query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
     logger.debug("SQL Query: %s", query_to_run)
+    res = EngineResults()
 
     with _connection.cursor() as cur:
         cur.execute(query_to_run, query_params)
-        results = []
         col_names = [i[0] for i in cur.description]
-        for res in cur:
+        for row in cur:
-            result = dict(zip(col_names, map(str, res)))
+            kvmap = dict(zip(col_names, map(str, row)))
-            result['template'] = result_template
+            res.add(res.types.KeyValue(kvmap=kvmap))
-            results.append(result)
+    return res
-        return results

searx/engines/meilisearch.py

@@ -33,15 +33,15 @@ Here is a simple example to query a Meilisearch instance:
 
 # pylint: disable=global-statement
 
-from json import loads, dumps
+from json import dumps
-
+from searx.result_types import EngineResults
+from searx.extended_types import SXNG_Response
 
 base_url = 'http://localhost:7700'
 index = ''
 auth_key = ''
 facet_filters = []
 _search_url = ''
-result_template = 'key-value.html'
 categories = ['general']
 paging = True
 
@@ -75,13 +75,12 @@ def request(query, params):
     return params
 
 
-def response(resp):
+def response(resp: SXNG_Response) -> EngineResults:
-    results = []
+    res = EngineResults()
 
-    resp_json = loads(resp.text)
+    resp_json = resp.json()
-    for result in resp_json['hits']:
+    for row in resp_json['hits']:
-        r = {key: str(value) for key, value in result.items()}
+        kvmap = {key: str(value) for key, value in row.items()}
-        r['template'] = result_template
+        res.add(res.types.KeyValue(kvmap=kvmap))
-        results.append(r)
 
-    return results
+    return res

searx/engines/mongodb.py

@@ -37,6 +37,7 @@ Implementations
 ===============
 
 """
+from __future__ import annotations
 
 import re
 
@@ -47,6 +48,8 @@ except ImportError:
     # to use the engine
     pass
 
+from searx.result_types import EngineResults
+
 
 engine_type = 'offline'
 
@@ -63,7 +66,6 @@ key = None
 paging = True
 results_per_page = 20
 exact_match_only = False
-result_template = 'key-value.html'
 
 _client = None
 
@@ -74,7 +76,7 @@ def init(_):
 
 def connect():
     global _client  # pylint: disable=global-statement
-    kwargs = {'port': port}
+    kwargs: dict[str, str | int] = {'port': port}
     if username:
         kwargs['username'] = username
     if password:
@@ -82,8 +84,8 @@ def connect():
     _client = MongoClient(host, **kwargs)[database][collection]
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
-    results = []
+    res = EngineResults()
     if exact_match_only:
         q = {'$eq': query}
     else:
@@ -92,11 +94,10 @@ def search(query, params):
 
     query = _client.find({key: q}).skip((params['pageno'] - 1) * results_per_page).limit(results_per_page)
 
-    results.append({'number_of_results': query.count()})
+    res.add(res.types.LegacyResult(number_of_results=query.count()))
-    for r in query:
+    for row in query:
-        del r['_id']
+        del row['_id']
-        r = {str(k): str(v) for k, v in r.items()}
+        kvmap = {str(k): str(v) for k, v in row.items()}
-        r['template'] = result_template
+        res.add(res.types.KeyValue(kvmap=kvmap))
-        results.append(r)
 
-    return results
+    return res

searx/engines/mysql_server.py

@@ -25,6 +25,8 @@ Implementations
 
 """
 
+from searx.result_types import EngineResults
+
 try:
     import mysql.connector  # type: ignore
 except ImportError:
@@ -55,7 +57,6 @@ query_str = ""
 
 limit = 10
 paging = True
-result_template = 'key-value.html'
 _connection = None
 
 
@@ -78,21 +79,15 @@ def init(engine_settings):
     )
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
+    res = EngineResults()
     query_params = {'query': query}
     query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
 
     with _connection.cursor() as cur:
         cur.execute(query_to_run, query_params)
+        for row in cur:
+            kvmap = dict(zip(cur.column_names, map(str, row)))
+            res.add(res.types.KeyValue(kvmap=kvmap))
 
-        return _fetch_results(cur)
+    return res
-
-
-def _fetch_results(cur):
-    results = []
-    for res in cur:
-        result = dict(zip(cur.column_names, map(str, res)))
-        result['template'] = result_template
-        results.append(result)
-
-    return results

searx/engines/postgresql.py

@@ -28,6 +28,8 @@ except ImportError:
     # manually to use the engine.
     pass
 
+from searx.result_types import EngineResults
+
 engine_type = 'offline'
 
 host = "127.0.0.1"
@@ -50,7 +52,6 @@ query_str = ""
 
 limit = 10
 paging = True
-result_template = 'key-value.html'
 _connection = None
 
 
@@ -72,7 +73,7 @@ def init(engine_settings):
     )
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
     query_params = {'query': query}
     query_to_run = query_str + ' LIMIT {0} OFFSET {1}'.format(limit, (params['pageno'] - 1) * limit)
 
@@ -82,20 +83,16 @@ def search(query, params):
             return _fetch_results(cur)
 
 
-def _fetch_results(cur):
+def _fetch_results(cur) -> EngineResults:
-    results = []
+    res = EngineResults()
-    titles = []
-
     try:
         titles = [column_desc.name for column_desc in cur.description]
-
+        for row in cur:
-        for res in cur:
+            kvmap = dict(zip(titles, map(str, row)))
-            result = dict(zip(titles, map(str, res)))
+            res.add(res.types.KeyValue(kvmap=kvmap))
-            result['template'] = result_template
-            results.append(result)
 
     # no results to fetch
     except psycopg2.ProgrammingError:
         pass
 
-    return results
+    return res

searx/engines/redis_server.py

@@ -36,6 +36,8 @@ Implementations
 
 import redis  # pylint: disable=import-error
 
+from searx.result_types import EngineResults
+
 engine_type = 'offline'
 
 # redis connection variables
@@ -46,7 +48,6 @@ db = 0
 
 # engine specific variables
 paging = False
-result_template = 'key-value.html'
 exact_match_only = True
 
 _redis_client = None
@@ -63,30 +64,25 @@ def init(_engine_settings):
     )
 
 
-def search(query, _params):
+def search(query, _params) -> EngineResults:
+    res = EngineResults()
+
     if not exact_match_only:
-        return search_keys(query)
+        for kvmap in search_keys(query):
+            res.add(res.types.KeyValue(kvmap=kvmap))
+        return res
 
-    ret = _redis_client.hgetall(query)
+    kvmap: dict[str, str] = _redis_client.hgetall(query)
-    if ret:
+    if kvmap:
-        ret['template'] = result_template
+        res.add(res.types.KeyValue(kvmap=kvmap))
-        return [ret]
+    elif " " in query:
-
+        qset, rest = query.split(" ", 1)
-    if ' ' in query:
+        for row in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)):
-        qset, rest = query.split(' ', 1)
+            res.add(res.types.KeyValue(kvmap={row[0]: row[1]}))
-        ret = []
+    return res
-        for res in _redis_client.hscan_iter(qset, match='*{}*'.format(rest)):
-            ret.append(
-                {
-                    res[0]: res[1],
-                    'template': result_template,
-                }
-            )
-        return ret
-    return []
 
 
-def search_keys(query):
+def search_keys(query) -> list[dict]:
     ret = []
     for key in _redis_client.scan_iter(match='*{}*'.format(query)):
         key_type = _redis_client.type(key)
@@ -98,7 +94,6 @@ def search_keys(query):
             res = dict(enumerate(_redis_client.lrange(key, 0, -1)))
 
         if res:
-            res['template'] = result_template
             res['redis_key'] = key
             ret.append(res)
     return ret

searx/engines/solr.py

@@ -29,9 +29,10 @@ This is an example configuration for searching in the collection
 
 # pylint: disable=global-statement
 
-from json import loads
 from urllib.parse import urlencode
 from searx.exceptions import SearxEngineAPIException
+from searx.result_types import EngineResults
+from searx.extended_types import SXNG_Response
 
 
 base_url = 'http://localhost:8983'
@@ -72,27 +73,21 @@ def request(query, params):
     return params
 
 
-def response(resp):
+def response(resp: SXNG_Response) -> EngineResults:
-    resp_json = __get_response(resp)
-
-    results = []
-    for result in resp_json['response']['docs']:
-        r = {key: str(value) for key, value in result.items()}
-        if len(r) == 0:
-            continue
-        r['template'] = 'key-value.html'
-        results.append(r)
-
-    return results
-
-
-def __get_response(resp):
     try:
-        resp_json = loads(resp.text)
+        resp_json = resp.json()
     except Exception as e:
         raise SearxEngineAPIException("failed to parse response") from e
 
-    if 'error' in resp_json:
+    if "error" in resp_json:
-        raise SearxEngineAPIException(resp_json['error']['msg'])
+        raise SearxEngineAPIException(resp_json["error"]["msg"])
 
-    return resp_json
+    res = EngineResults()
+
+    for result in resp_json["response"]["docs"]:
+        kvmap = {key: str(value) for key, value in result.items()}
+        if not kvmap:
+            continue
+        res.add(res.types.KeyValue(kvmap=kvmap))
+
+    return res

searx/engines/sqlite.py

@@ -2,6 +2,14 @@
 """SQLite is a small, fast and reliable SQL database engine.  It does not require
 any extra dependency.
 
+Configuration
+=============
+
+The engine has the following (additional) settings:
+
+- :py:obj:`result_type`
+
+
 Example
 =======
 
@@ -18,29 +26,32 @@ Query to test: ``!mediathekview concert``
 
 .. code:: yaml
 
-   - name: mediathekview
+  - name: mediathekview
-     engine: sqlite
+    engine: sqlite
-     disabled: False
+    shortcut: mediathekview
-     categories: general
+    categories: [general, videos]
-     result_template: default.html
+    result_type: MainResult
-     database: searx/data/filmliste-v2.db
+    database: searx/data/filmliste-v2.db
-     query_str:  >-
+    query_str: >-
-       SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
+      SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
-              COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
+             COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
-              description AS content
+             description AS content
-         FROM film
+        FROM film
-        WHERE title LIKE :wildcard OR description LIKE :wildcard
+       WHERE title LIKE :wildcard OR description LIKE :wildcard
-        ORDER BY duration DESC
+       ORDER BY duration DESC
 
 Implementations
 ===============
 
 """
-
+import typing
 import sqlite3
 import contextlib
 
-engine_type = 'offline'
+from searx.result_types import EngineResults
+from searx.result_types import MainResult, KeyValue
+
+engine_type = "offline"
 
 database = ""
 """Filename of the SQLite DB."""
@@ -48,9 +59,11 @@ database = ""
 query_str = ""
 """SQL query that returns the result items."""
 
+result_type: typing.Literal["MainResult", "KeyValue"] = "KeyValue"
+"""The result type can be :py:obj:`MainResult` or :py:obj:`KeyValue`."""
+
 limit = 10
 paging = True
-result_template = 'key-value.html'
 
 
 def init(engine_settings):
@@ -80,9 +93,8 @@ def sqlite_cursor():
             yield cursor
 
 
-def search(query, params):
+def search(query, params) -> EngineResults:
-    results = []
+    res = EngineResults()
-
     query_params = {
         'query': query,
         'wildcard': r'%' + query.replace(' ', r'%') + r'%',
@@ -97,9 +109,11 @@ def search(query, params):
         col_names = [cn[0] for cn in cur.description]
 
         for row in cur.fetchall():
-            item = dict(zip(col_names, map(str, row)))
+            kvmap = dict(zip(col_names, map(str, row)))
-            item['template'] = result_template
+            if result_type == "MainResult":
-            logger.debug("append result --> %s", item)
+                item = MainResult(**kvmap)  # type: ignore
-            results.append(item)
+            else:
+                item = KeyValue(kvmap=kvmap)
+            res.add(item)
 
-    return results
+    return res

searx/plugins/tor_check.py

@@ -21,7 +21,8 @@ from flask_babel import gettext
 from httpx import HTTPError
 
 from searx.network import get
-from searx.result_types import Answer
+from searx.result_types import EngineResults
+from searx.botdetection import get_real_ip
 
 
 default_on = False
@@ -51,8 +52,8 @@ url_exit_list = "https://check.torproject.org/exit-addresses"
 """URL to load Tor exit list from."""
 
 
-def post_search(request, search) -> list[Answer]:
+def post_search(request, search) -> EngineResults:
-    results = []
+    results = EngineResults()
 
     if search.search_query.pageno > 1:
         return results
@@ -67,22 +68,17 @@ def post_search(request, search) -> list[Answer]:
         except HTTPError:
             # No answer, return error
             msg = gettext("Could not download the list of Tor exit-nodes from")
-            Answer(results=results, answer=f"{msg} {url_exit_list}")
+            results.add(results.types.Answer(answer=f"{msg} {url_exit_list}"))
             return results
 
-        x_forwarded_for = request.headers.getlist("X-Forwarded-For")
+        real_ip = get_real_ip(request)
 
-        if x_forwarded_for:
+        if real_ip in node_list:
-            ip_address = x_forwarded_for[0]
-        else:
-            ip_address = request.remote_addr
-
-        if ip_address in node_list:
             msg = gettext("You are using Tor and it looks like you have the external IP address")
-            Answer(results=results, answer=f"{msg} {ip_address}")
+            results.add(results.types.Answer(answer=f"{msg} {real_ip}"))
 
         else:
             msg = gettext("You are not using Tor and you have the external IP address")
-            Answer(results=results, answer=f"{msg} {ip_address}")
+            results.add(results.types.Answer(answer=f"{msg} {real_ip}"))
 
     return results

searx/plugins/unit_converter.py

@@ -25,7 +25,7 @@ import babel.numbers
 from flask_babel import gettext, get_locale
 
 from searx import data
-from searx.result_types import Answer
+from searx.result_types import EngineResults
 
 
 name = "Unit converter plugin"
@@ -245,8 +245,8 @@ def _parse_text_and_convert(from_query, to_query) -> str | None:
     return f'{result} {target_symbol}'
 
 
-def post_search(_request, search) -> list[Answer]:
+def post_search(_request, search) -> EngineResults:
-    results = []
+    results = EngineResults()
 
     # only convert between units on the first page
     if search.search_query.pageno > 1:
@@ -264,6 +264,6 @@ def post_search(_request, search) -> list[Answer]:
                 from_query, to_query = query.split(keyword, 1)
                 target_val = _parse_text_and_convert(from_query.strip(), to_query.strip())
                 if target_val:
-                    Answer(results=results, answer=target_val)
+                    results.add(results.types.Answer(answer=target_val))
 
     return results

searx/result_types/__init__.py

@@ -13,14 +13,15 @@
 
 from __future__ import annotations
 
-__all__ = ["Result", "EngineResults", "AnswerSet", "Answer", "Translations"]
+__all__ = ["Result", "MainResult", "KeyValue", "EngineResults", "AnswerSet", "Answer", "Translations"]
 
 import abc
 
 from searx import enginelib
 
-from ._base import Result, LegacyResult
+from ._base import Result, MainResult, LegacyResult
 from .answer import AnswerSet, Answer, Translations
+from .keyvalue import KeyValue
 
 
 class ResultList(list, abc.ABC):
@@ -30,13 +31,19 @@ class ResultList(list, abc.ABC):
         """The collection of result types (which have already been implemented)."""
 
         Answer = Answer
+        KeyValue = KeyValue
+        MainResult = MainResult
+        Result = Result
         Translations = Translations
 
+        # for backward compatibility
+        LegacyResult = LegacyResult
+
     def __init__(self):
         # pylint: disable=useless-parent-delegation
         super().__init__()
 
-    def add(self, result: Result):
+    def add(self, result: Result | LegacyResult):
         """Add a :py:`Result` item to the result list."""
         self.append(result)
 

searx/result_types/_base.py

@@ -10,6 +10,8 @@
 .. autoclass:: Result
    :members:
 
+.. _LegacyResult:
+
 .. autoclass:: LegacyResult
    :members:
 """
@@ -22,9 +24,88 @@ __all__ = ["Result"]
 import re
 import urllib.parse
 import warnings
+import typing
 
 import msgspec
 
+from searx import logger as log
+
+WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
+
+
+def _normalize_url_fields(result: Result | LegacyResult):
+
+    # As soon we need LegacyResult not any longer, we can move this function to
+    # method Result.normalize_result_fields
+
+    if result.url and not result.parsed_url:
+        if not isinstance(result.url, str):
+            log.debug('result: invalid URL: %s', str(result))
+            result.url = ""
+            result.parsed_url = None
+        else:
+            result.parsed_url = urllib.parse.urlparse(result.url)
+
+    if result.parsed_url:
+        result.parsed_url = result.parsed_url._replace(
+            # if the result has no scheme, use http as default
+            scheme=result.parsed_url.scheme or "http",
+            # normalize ``www.example.com`` to ``example.com``
+            netloc=result.parsed_url.netloc.replace("www.", ""),
+            # normalize ``example.com/path/`` to ``example.com/path``
+            path=result.parsed_url.path.rstrip("/"),
+        )
+        result.url = result.parsed_url.geturl()
+
+    if isinstance(result, LegacyResult) and getattr(result, "infobox", None):
+        # As soon we have InfoboxResult, we can move this function to method
+        # InfoboxResult.normalize_result_fields
+
+        infobox_urls: list[dict[str, str]] = getattr(result, "urls", [])
+        for item in infobox_urls:
+            _url = item.get("url")
+            if not _url:
+                continue
+            _url = urllib.parse.urlparse(_url)
+            item["url"] = _url._replace(
+                scheme=_url.scheme or "http",
+                netloc=_url.netloc.replace("www.", ""),
+                path=_url.path.rstrip("/"),
+            ).geturl()
+
+        infobox_id = getattr(result, "id", None)
+        if infobox_id:
+            _url = urllib.parse.urlparse(infobox_id)
+            result.id = _url._replace(
+                scheme=_url.scheme or "http",
+                netloc=_url.netloc.replace("www.", ""),
+                path=_url.path.rstrip("/"),
+            ).geturl()
+
+
+def _normalize_text_fields(result: MainResult | LegacyResult):
+
+    # As soon we need LegacyResult not any longer, we can move this function to
+    # method MainResult.normalize_result_fields
+
+    # Actually, a type check should not be necessary if the engine is
+    # implemented correctly. Historically, however, we have always had a type
+    # check here.
+
+    if result.title and not isinstance(result.title, str):
+        log.debug("result: invalid type of field 'title': %s", str(result))
+        result.title = str(result)
+    if result.content and not isinstance(result.content, str):
+        log.debug("result: invalid type of field 'content': %s", str(result))
+        result.content = str(result)
+
+    # normalize title and content
+    result.title = WHITESPACE_REGEX.sub(" ", result.title).strip()
+    result.content = WHITESPACE_REGEX.sub(" ", result.content).strip()
+    if result.content == result.title:
+        # avoid duplicate content between the content and title fields
+        result.content = ""
+
 
 class Result(msgspec.Struct, kw_only=True):
     """Base class of all result types :ref:`result types`."""
@@ -54,21 +135,20 @@ class Result(msgspec.Struct, kw_only=True):
     """
 
     def normalize_result_fields(self):
-        """Normalize a result ..
+        """Normalize fields ``url`` and ``parse_sql``.
 
-        - if field ``url`` is set and field ``parse_url`` is unset, init
+        - If field ``url`` is set and field ``parse_url`` is unset, init
-          ``parse_url`` from field ``url``.  This method can be extended in the
+          ``parse_url`` from field ``url``.  The ``url`` field is initialized
-          inheritance.
+          with the resulting value in ``parse_url``, if ``url`` and
+          ``parse_url`` are not equal.
 
+        - ``www.example.com`` and ``example.com`` are equivalent and are normalized
+          to ``example.com``.
+
+        - ``example.com/path/`` and ``example.com/path`` are equivalent and are
+          normalized to ``example.com/path``.
         """
-
+        _normalize_url_fields(self)
-        if not self.parsed_url and self.url:
-            self.parsed_url = urllib.parse.urlparse(self.url)
-
-            # if the result has no scheme, use http as default
-            if not self.parsed_url.scheme:
-                self.parsed_url = self.parsed_url._replace(scheme="http")
-                self.url = self.parsed_url.geturl()
 
     def __post_init__(self):
         pass
@@ -84,7 +164,6 @@ class Result(msgspec.Struct, kw_only=True):
         The hash value is used in contexts, e.g. when checking for equality to
         identify identical results from different sources (engines).
         """
-
         return id(self)
 
     def __eq__(self, other):
@@ -113,12 +192,19 @@ class Result(msgspec.Struct, kw_only=True):
     def as_dict(self):
         return {f: getattr(self, f) for f in self.__struct_fields__}
 
+    def defaults_from(self, other: Result):
+        """Fields not set in *self* will be updated from the field values of the
+        *other*.
+        """
+        for field_name in self.__struct_fields__:
+            self_val = getattr(self, field_name, False)
+            other_val = getattr(other, field_name, False)
+            if self_val:
+                setattr(self, field_name, other_val)
+
 
 class MainResult(Result):  # pylint: disable=missing-class-docstring
-
+    """Base class of all result types displayed in :ref:`area main results`."""
-    # open_group and close_group should not manged in the Result class (we should rop it from here!)
-    open_group: bool = False
-    close_group: bool = False
 
     title: str = ""
     """Link title of the result item."""
@@ -132,6 +218,43 @@ class MainResult(Result):  # pylint: disable=missing-class-docstring
     thumbnail: str = ""
     """URL of a thumbnail that is displayed in the result item."""
 
+    priority: typing.Literal["", "high", "low"] = ""
+    """The priority can be set via :ref:`hostnames plugin`, for example."""
+
+    engines: set[str] = set()
+    """In a merged results list, the names of the engines that found this result
+    are listed in this field."""
+
+    # open_group and close_group should not manged in the Result
+    # class (we should drop it from here!)
+    open_group: bool = False
+    close_group: bool = False
+    positions: list[int] = []
+    score: float = 0
+    category: str = ""
+
+    def __hash__(self) -> int:
+        """Ordinary url-results are equal if their values for
+        :py:obj:`Result.template`, :py:obj:`Result.parsed_url` (without scheme)
+        and :py:obj:`MainResult.img_src` are equal.
+        """
+        if not self.parsed_url:
+            raise ValueError(f"missing a value in field 'parsed_url': {self}")
+
+        url = self.parsed_url
+        return hash(
+            f"{self.template}"
+            + f"|{url.netloc}|{url.path}|{url.params}|{url.query}|{url.fragment}"
+            + f"|{self.img_src}"
+        )
+
+    def normalize_result_fields(self):
+        super().normalize_result_fields()
+
+        _normalize_text_fields(self)
+        if self.engine:
+            self.engines.add(self.engine)
+
 
 class LegacyResult(dict):
     """A wrapper around a legacy result item.  The SearXNG core uses this class
@@ -150,7 +273,27 @@ class LegacyResult(dict):
     """
 
     UNSET = object()
-    WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)
+
+    # emulate field types from type class Result
+    url: str | None
+    template: str
+    engine: str
+    parsed_url: urllib.parse.ParseResult | None
+
+    # emulate field types from type class MainResult
+    title: str
+    content: str
+    img_src: str
+    thumbnail: str
+    priority: typing.Literal["", "high", "low"]
+    engines: set[str]
+    positions: list[int]
+    score: float
+    category: str
+
+    # infobox result
+    urls: list[dict[str, str]]
+    attributes: list[dict[str, str]]
 
     def as_dict(self):
         return self
@@ -159,14 +302,26 @@ class LegacyResult(dict):
 
         super().__init__(*args, **kwargs)
 
-        # Init fields with defaults / compare with defaults of the fields in class Result
+        # emulate field types from type class Result
-        self.engine = self.get("engine", "")
+        self["url"] = self.get("url")
-        self.template = self.get("template", "default.html")
+        self["template"] = self.get("template", "default.html")
-        self.url = self.get("url", None)
+        self["engine"] = self.get("engine", "")
-        self.parsed_url = self.get("parsed_url", None)
+        self["parsed_url"] = self.get("parsed_url")
 
-        self.content = self.get("content", "")
+        # emulate field types from type class MainResult
-        self.title = self.get("title", "")
+        self["title"] = self.get("title", "")
+        self["content"] = self.get("content", "")
+        self["img_src"] = self.get("img_src", "")
+        self["thumbnail"] = self.get("thumbnail", "")
+        self["priority"] = self.get("priority", "")
+        self["engines"] = self.get("engines", set())
+        self["positions"] = self.get("positions", "")
+        self["score"] = self.get("score", 0)
+        self["category"] = self.get("category", "")
+
+        if "infobox" in self:
+            self["urls"] = self.get("urls", [])
+            self["attributes"] = self.get("attributes", [])
 
         # Legacy types that have already been ported to a type ..
 
@@ -178,13 +333,47 @@ class LegacyResult(dict):
             )
             self.template = "answer/legacy.html"
 
+        if self.template == "keyvalue.html":
+            warnings.warn(
+                f"engine {self.engine} is using deprecated `dict` for key/value results"
+                f" / use a class from searx.result_types",
+                DeprecationWarning,
+            )
+
+    def __getattr__(self, name: str, default=UNSET) -> typing.Any:
+        if default == self.UNSET and name not in self:
+            raise AttributeError(f"LegacyResult object has no field named: {name}")
+        return self[name]
+
+    def __setattr__(self, name: str, val):
+        self[name] = val
+
     def __hash__(self) -> int:  # type: ignore
 
         if "answer" in self:
+            # deprecated ..
             return hash(self["answer"])
+
+        if self.template == "images.html":
+            # image results are equal if their values for template, the url and
+            # the img_src are equal.
+            return hash(f"{self.template}|{self.url}|{self.img_src}")
+
         if not any(cls in self for cls in ["suggestion", "correction", "infobox", "number_of_results", "engine_data"]):
-            # it is a commun url-result ..
+            # Ordinary url-results are equal if their values for template,
-            return hash(self.url)
+            # parsed_url (without schema) and img_src` are equal.
+
+            # Code copied from with MainResult.__hash__:
+            if not self.parsed_url:
+                raise ValueError(f"missing a value in field 'parsed_url': {self}")
+
+            url = self.parsed_url
+            return hash(
+                f"{self.template}"
+                + f"|{url.netloc}|{url.path}|{url.params}|{url.query}|{url.fragment}"
+                + f"|{self.img_src}"
+            )
+
         return id(self)
 
     def __eq__(self, other):
@@ -195,30 +384,13 @@ class LegacyResult(dict):
 
         return f"LegacyResult: {super().__repr__()}"
 
-    def __getattr__(self, name: str, default=UNSET):
-
-        if default == self.UNSET and name not in self:
-            raise AttributeError(f"LegacyResult object has no field named: {name}")
-        return self[name]
-
-    def __setattr__(self, name: str, val):
-
-        self[name] = val
-
     def normalize_result_fields(self):
+        _normalize_url_fields(self)
+        _normalize_text_fields(self)
+        if self.engine:
+            self.engines.add(self.engine)
 
-        self.title = self.WHITESPACE_REGEX.sub(" ", self.title)
+    def defaults_from(self, other: LegacyResult):
-
+        for k, v in other.items():
-        if not self.parsed_url and self.url:
+            if not self.get(k):
-            self.parsed_url = urllib.parse.urlparse(self.url)
+                self[k] = v
-
-            # if the result has no scheme, use http as default
-            if not self.parsed_url.scheme:
-                self.parsed_url = self.parsed_url._replace(scheme="http")
-                self.url = self.parsed_url.geturl()
-
-        if self.content:
-            self.content = self.WHITESPACE_REGEX.sub(" ", self.content)
-            if self.content == self.title:
-                # avoid duplicate content between the content and title fields
-                self.content = ""

searx/result_types/keyvalue.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""
+Typification of the *keyvalue* results.  Results of this type are rendered in
+the :origin:`keyvalue.html <searx/templates/simple/result_templates/keyvalue.html>`
+template.
+
+----
+
+.. autoclass:: KeyValue
+   :members:
+   :show-inheritance:
+
+"""
+# pylint: disable=too-few-public-methods
+
+from __future__ import annotations
+
+__all__ = ["KeyValue"]
+
+import typing
+from collections import OrderedDict
+
+from ._base import MainResult
+
+
+class KeyValue(MainResult, kw_only=True):
+    """Simple table view which maps *key* names (first col) to *values*
+    (second col)."""
+
+    template: str = "keyvalue.html"
+
+    kvmap: dict[str, typing.Any] | OrderedDict[str, typing.Any]
+    """Dictionary with keys and values. To sort keys, use :py:obj:`OrderedDict`."""
+
+    caption: str = ""
+    """Optional caption for this result."""
+
+    key_title: str = ""
+    """Optional title for the *key column*."""
+
+    value_title: str = ""
+    """Optional title for the *value column*."""
+
+    def __hash__(self) -> int:
+        """The KeyValues objects are checked for object identity, even if all
+        fields of two results have the same values, they are different from each
+        other.
+        """
+        return id(self)

searx/results.py

@@ -1,143 +1,25 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# pylint: disable=missing-module-docstring
+# pylint: disable=missing-module-docstring, missing-class-docstring
 from __future__ import annotations
 
 import warnings
-import re
 from collections import defaultdict
-from operator import itemgetter
 from threading import RLock
 from typing import List, NamedTuple, Set
-from urllib.parse import urlparse, unquote
 
-from searx import logger
+from searx import logger as log
-from searx.engines import engines
+import searx.engines
-from searx.metrics import histogram_observe, counter_add, count_error
+from searx.metrics import histogram_observe, counter_add
-
+from searx.result_types import Result, LegacyResult, MainResult
-from searx.result_types import Result, LegacyResult
 from searx.result_types.answer import AnswerSet, BaseAnswer
 
-CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
 
-
+def calculate_score(result, priority) -> float:
-# return the meaningful length of the content for a result
-def result_content_len(content):
-    if isinstance(content, str):
-        return len(CONTENT_LEN_IGNORED_CHARS_REGEX.sub('', content))
-    return 0
-
-
-def compare_urls(url_a, url_b):
-    """Lazy compare between two URL.
-    "www.example.com" and "example.com" are equals.
-    "www.example.com/path/" and "www.example.com/path" are equals.
-    "https://www.example.com/" and "http://www.example.com/" are equals.
-
-    Args:
-        url_a (ParseResult): first URL
-        url_b (ParseResult): second URL
-
-    Returns:
-        bool: True if url_a and url_b are equals
-    """
-    # ignore www. in comparison
-    if url_a.netloc.startswith('www.'):
-        host_a = url_a.netloc.replace('www.', '', 1)
-    else:
-        host_a = url_a.netloc
-    if url_b.netloc.startswith('www.'):
-        host_b = url_b.netloc.replace('www.', '', 1)
-    else:
-        host_b = url_b.netloc
-
-    if host_a != host_b or url_a.query != url_b.query or url_a.fragment != url_b.fragment:
-        return False
-
-    # remove / from the end of the url if required
-    path_a = url_a.path[:-1] if url_a.path.endswith('/') else url_a.path
-    path_b = url_b.path[:-1] if url_b.path.endswith('/') else url_b.path
-
-    return unquote(path_a) == unquote(path_b)
-
-
-def merge_two_infoboxes(infobox1, infobox2):  # pylint: disable=too-many-branches, too-many-statements
-    # get engines weights
-    if hasattr(engines[infobox1['engine']], 'weight'):
-        weight1 = engines[infobox1['engine']].weight
-    else:
-        weight1 = 1
-    if hasattr(engines[infobox2['engine']], 'weight'):
-        weight2 = engines[infobox2['engine']].weight
-    else:
-        weight2 = 1
-
-    if weight2 > weight1:
-        infobox1['engine'] = infobox2['engine']
-
-    infobox1['engines'] |= infobox2['engines']
-
-    if 'urls' in infobox2:
-        urls1 = infobox1.get('urls', None)
-        if urls1 is None:
-            urls1 = []
-
-        for url2 in infobox2.get('urls', []):
-            unique_url = True
-            parsed_url2 = urlparse(url2.get('url', ''))
-            entity_url2 = url2.get('entity')
-            for url1 in urls1:
-                if (entity_url2 is not None and url1.get('entity') == entity_url2) or compare_urls(
-                    urlparse(url1.get('url', '')), parsed_url2
-                ):
-                    unique_url = False
-                    break
-            if unique_url:
-                urls1.append(url2)
-
-        infobox1['urls'] = urls1
-
-    if 'img_src' in infobox2:
-        img1 = infobox1.get('img_src', None)
-        img2 = infobox2.get('img_src')
-        if img1 is None:
-            infobox1['img_src'] = img2
-        elif weight2 > weight1:
-            infobox1['img_src'] = img2
-
-    if 'attributes' in infobox2:
-        attributes1 = infobox1.get('attributes')
-        if attributes1 is None:
-            infobox1['attributes'] = attributes1 = []
-
-        attributeSet = set()
-        for attribute in attributes1:
-            label = attribute.get('label')
-            if label not in attributeSet:
-                attributeSet.add(label)
-            entity = attribute.get('entity')
-            if entity not in attributeSet:
-                attributeSet.add(entity)
-
-        for attribute in infobox2.get('attributes', []):
-            if attribute.get('label') not in attributeSet and attribute.get('entity') not in attributeSet:
-                attributes1.append(attribute)
-
-    if 'content' in infobox2:
-        content1 = infobox1.get('content', None)
-        content2 = infobox2.get('content', '')
-        if content1 is not None:
-            if result_content_len(content2) > result_content_len(content1):
-                infobox1['content'] = content2
-        else:
-            infobox1['content'] = content2
-
-
-def result_score(result, priority):
     weight = 1.0
 
     for result_engine in result['engines']:
-        if hasattr(engines.get(result_engine), 'weight'):
+        if hasattr(searx.engines.engines.get(result_engine), 'weight'):
-            weight *= float(engines[result_engine].weight)
+            weight *= float(searx.engines.engines[result_engine].weight)
 
     weight *= len(result['positions'])
     score = 0
@@ -153,61 +35,53 @@ def result_score(result, priority):
     return score
 
 
-class Timing(NamedTuple):  # pylint: disable=missing-class-docstring
+class Timing(NamedTuple):
     engine: str
     total: float
     load: float
 
 
-class UnresponsiveEngine(NamedTuple):  # pylint: disable=missing-class-docstring
+class UnresponsiveEngine(NamedTuple):
     engine: str
     error_type: str
     suspended: bool
 
 
 class ResultContainer:
-    """docstring for ResultContainer"""
+    """In the result container, the results are collected, sorted and duplicates
+    will be merged."""
 
-    __slots__ = (
+    # pylint: disable=too-many-statements
-        '_merged_results',
+
-        'infoboxes',
+    main_results_map: dict[int, MainResult | LegacyResult]
-        'suggestions',
+    infoboxes: list[LegacyResult]
-        'answers',
+    suggestions: set[str]
-        'corrections',
+    answers: AnswerSet
-        '_number_of_results',
+    corrections: set[str]
-        '_closed',
-        'paging',
-        'unresponsive_engines',
-        'timings',
-        'redirect_url',
-        'engine_data',
-        'on_result',
-        '_lock',
-    )
 
     def __init__(self):
-        super().__init__()
+        self.main_results_map = {}
-        self._merged_results: list[LegacyResult] = []
+        self.infoboxes = []
-        self.infoboxes: list[dict] = []
+        self.suggestions = set()
-        self.suggestions: set[str] = set()
         self.answers = AnswerSet()
         self.corrections = set()
+
         self._number_of_results: list[int] = []
-        self.engine_data: dict[str, str | dict] = defaultdict(dict)
+        self.engine_data: dict[str, dict[str, str]] = defaultdict(dict)
         self._closed: bool = False
         self.paging: bool = False
         self.unresponsive_engines: Set[UnresponsiveEngine] = set()
         self.timings: List[Timing] = []
-        self.redirect_url = None
+        self.redirect_url: str | None = None
         self.on_result = lambda _: True
         self._lock = RLock()
+        self._main_results_sorted: list[MainResult | LegacyResult] = None  # type: ignore
 
     def extend(self, engine_name: str | None, results):  # pylint: disable=too-many-branches
         if self._closed:
+            log.debug("container is closed, ignoring results: %s", results)
             return
-
+        main_count = 0
-        standard_result_count = 0
-        error_msgs = set()
 
         for result in list(results):
 
@@ -217,267 +91,284 @@ class ResultContainer:
 
                 if isinstance(result, BaseAnswer) and self.on_result(result):
                     self.answers.add(result)
+                elif isinstance(result, MainResult) and self.on_result(result):
+                    main_count += 1
+                    self._merge_main_result(result, main_count)
                 else:
                     # more types need to be implemented in the future ..
                     raise NotImplementedError(f"no handler implemented to process the result of type {result}")
 
             else:
-                result['engine'] = result.get('engine') or engine_name or ""
+                result["engine"] = result.get("engine") or engine_name or ""
                 result = LegacyResult(result)  # for backward compatibility, will be romeved one day
+                result.normalize_result_fields()
 
-                if 'suggestion' in result and self.on_result(result):
+                if "suggestion" in result:
-                    self.suggestions.add(result['suggestion'])
+                    if self.on_result(result):
-                elif 'answer' in result and self.on_result(result):
+                        self.suggestions.add(result["suggestion"])
-                    warnings.warn(
+                    continue
-                        f"answer results from engine {result.engine}"
-                        " are without typification / migrate to Answer class.",
-                        DeprecationWarning,
-                    )
-                    self.answers.add(result)
-                elif 'correction' in result and self.on_result(result):
-                    self.corrections.add(result['correction'])
-                elif 'infobox' in result and self.on_result(result):
-                    self._merge_infobox(result)
-                elif 'number_of_results' in result and self.on_result(result):
-                    self._number_of_results.append(result['number_of_results'])
-                elif 'engine_data' in result and self.on_result(result):
-                    self.engine_data[result.engine][result['key']] = result['engine_data']
-                elif result.url:
-                    # standard result (url, title, content)
-                    if not self._is_valid_url_result(result, error_msgs):
-                        continue
-                    # normalize the result
-                    result.normalize_result_fields()
-                    # call on_result call searx.search.SearchWithPlugins._on_result
-                    # which calls the plugins
-                    if not self.on_result(result):
-                        continue
-                    self.__merge_url_result(result, standard_result_count + 1)
-                    standard_result_count += 1
-                elif self.on_result(result):
-                    self.__merge_result_no_url(result, standard_result_count + 1)
-                    standard_result_count += 1
 
-        if len(error_msgs) > 0:
+                if "answer" in result:
-            for msg in error_msgs:
+                    if self.on_result(result):
-                count_error(engine_name, 'some results are invalids: ' + msg, secondary=True)
+                        warnings.warn(
+                            f"answer results from engine {result.engine}"
+                            " are without typification / migrate to Answer class.",
+                            DeprecationWarning,
+                        )
+                        self.answers.add(result)  # type: ignore
+                    continue
 
-        if engine_name in engines:
+                if "correction" in result:
-            histogram_observe(standard_result_count, 'engine', engine_name, 'result', 'count')
+                    if self.on_result(result):
+                        self.corrections.add(result["correction"])
+                    continue
 
-        if not self.paging and engine_name in engines and engines[engine_name].paging:
+                if "infobox" in result:
-            self.paging = True
+                    if self.on_result(result):
+                        self._merge_infobox(result)
+                    continue
 
-    def _merge_infobox(self, infobox):
+                if "number_of_results" in result:
+                    if self.on_result(result):
+                        self._number_of_results.append(result["number_of_results"])
+                    continue
+
+                if "engine_data" in result:
+                    if self.on_result(result):
+                        if result.engine:
+                            self.engine_data[result.engine][result["key"]] = result["engine_data"]
+                    continue
+
+                if self.on_result(result):
+                    main_count += 1
+                    self._merge_main_result(result, main_count)
+                    continue
+
+        if engine_name in searx.engines.engines:
+            eng = searx.engines.engines[engine_name]
+            histogram_observe(main_count, "engine", eng.name, "result", "count")
+            if not self.paging and eng.paging:
+                self.paging = True
+
+    def _merge_infobox(self, new_infobox: LegacyResult):
         add_infobox = True
-        infobox_id = infobox.get('id', None)
+
-        infobox['engines'] = set([infobox['engine']])
+        new_id = getattr(new_infobox, "id", None)
-        if infobox_id is not None:
+        if new_id is not None:
-            parsed_url_infobox_id = urlparse(infobox_id)
             with self._lock:
-                for existingIndex in self.infoboxes:
+                for existing_infobox in self.infoboxes:
-                    if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):
+                    if new_id == getattr(existing_infobox, "id", None):
-                        merge_two_infoboxes(existingIndex, infobox)
+                        merge_two_infoboxes(existing_infobox, new_infobox)
                         add_infobox = False
-
         if add_infobox:
-            self.infoboxes.append(infobox)
+            self.infoboxes.append(new_infobox)
 
-    def _is_valid_url_result(self, result, error_msgs):
+    def _merge_main_result(self, result: MainResult | LegacyResult, position):
-        if 'url' in result:
+        result_hash = hash(result)
-            if not isinstance(result['url'], str):
-                logger.debug('result: invalid URL: %s', str(result))
-                error_msgs.add('invalid URL')
-                return False
 
-        if 'title' in result and not isinstance(result['title'], str):
-            logger.debug('result: invalid title: %s', str(result))
-            error_msgs.add('invalid title')
-            return False
-
-        if 'content' in result:
-            if not isinstance(result['content'], str):
-                logger.debug('result: invalid content: %s', str(result))
-                error_msgs.add('invalid content')
-                return False
-
-        return True
-
-    def __merge_url_result(self, result, position):
-        result['engines'] = set([result['engine']])
         with self._lock:
-            duplicated = self.__find_duplicated_http_result(result)
+
-            if duplicated:
+            merged = self.main_results_map.get(result_hash)
-                self.__merge_duplicated_http_result(duplicated, result, position)
+            if not merged:
+                # if there is no duplicate in the merged results, append result
+                result.positions = [position]
+                self.main_results_map[result_hash] = result
                 return
 
-            # if there is no duplicate found, append result
+            merge_two_main_results(merged, result)
-            result['positions'] = [position]
+            # add the new position
-            self._merged_results.append(result)
+            merged.positions.append(position)
-
-    def __find_duplicated_http_result(self, result):
-        result_template = result.get('template')
-        for merged_result in self._merged_results:
-            if not merged_result.get('parsed_url'):
-                continue
-
-            if compare_urls(result['parsed_url'], merged_result['parsed_url']) and result_template == merged_result.get(
-                'template'
-            ):
-                if result_template != 'images.html':
-                    # not an image, same template, same url : it's a duplicate
-                    return merged_result
-
-                # it's an image
-                # it's a duplicate if the parsed_url, template and img_src are different
-                if result.get('img_src', '') == merged_result.get('img_src', ''):
-                    return merged_result
-        return None
-
-    def __merge_duplicated_http_result(self, duplicated, result, position):
-        # use content with more text
-        if result_content_len(result.get('content', '')) > result_content_len(duplicated.get('content', '')):
-            duplicated['content'] = result['content']
-
-        # use title with more text
-        if result_content_len(result.get('title', '')) > len(duplicated.get('title', '')):
-            duplicated['title'] = result['title']
-
-        # merge all result's parameters not found in duplicate
-        for key in result.keys():
-            if not duplicated.get(key):
-                duplicated[key] = result.get(key)
-
-        # add the new position
-        duplicated['positions'].append(position)
-
-        # add engine to list of result-engines
-        duplicated['engines'].add(result['engine'])
-
-        # use https if possible
-        if duplicated['parsed_url'].scheme != 'https' and result['parsed_url'].scheme == 'https':
-            duplicated['url'] = result['parsed_url'].geturl()
-            duplicated['parsed_url'] = result['parsed_url']
-
-    def __merge_result_no_url(self, result, position):
-        result['engines'] = set([result['engine']])
-        result['positions'] = [position]
-        with self._lock:
-            self._merged_results.append(result)
 
     def close(self):
         self._closed = True
 
-        for result in self._merged_results:
+        for result in self.main_results_map.values():
-            result['score'] = result_score(result, result.get('priority'))
+            result.score = calculate_score(result, result.priority)
-            # removing html content and whitespace duplications
+            for eng_name in result.engines:
-            if result.get('content'):
+                counter_add(result.score, 'engine', eng_name, 'score')
-                result['content'] = result['content'].strip()
-            if result.get('title'):
-                result['title'] = ' '.join(result['title'].strip().split())
 
-            for result_engine in result['engines']:
+    def get_ordered_results(self) -> list[MainResult | LegacyResult]:
-                counter_add(result['score'], 'engine', result_engine, 'score')
+        """Returns a sorted list of results to be displayed in the main result
+        area (:ref:`result types`)."""
 
-        results = sorted(self._merged_results, key=itemgetter('score'), reverse=True)
+        if not self._closed:
+            self.close()
+
+        if self._main_results_sorted:
+            return self._main_results_sorted
+
+        # first pass, sort results by "score" (descanding)
+        results = sorted(self.main_results_map.values(), key=lambda x: x.score, reverse=True)
 
         # pass 2 : group results by category and template
         gresults = []
         categoryPositions = {}
+        max_count = 8
+        max_distance = 20
 
         for res in results:
-            if not res.get('url'):
+            # do we need to handle more than one category per engine?
-                continue
+            engine = searx.engines.engines.get(res.engine or "")
+            if engine:
+                res.category = engine.categories[0] if len(engine.categories) > 0 else ""
 
             # do we need to handle more than one category per engine?
-            engine = engines[res['engine']]
+            category = f"{res.category}:{res.template}:{'img_src' if (res.thumbnail or res.img_src) else ''}"
-            res['category'] = engine.categories[0] if len(engine.categories) > 0 else ''
+            grp = categoryPositions.get(category)
 
-            # do we need to handle more than one category per engine?
+            # group with previous results using the same category, if the group
-            category = (
+            # can accept more result and is not too far from the current
-                res['category']
+            # position
-                + ':'
-                + res.get('template', '')
-                + ':'
-                + ('img_src' if 'img_src' in res or 'thumbnail' in res else '')
-            )
 
-            current = None if category not in categoryPositions else categoryPositions[category]
+            if (grp is not None) and (grp["count"] > 0) and (len(gresults) - grp["index"] < max_distance):
-
+                # group with the previous results using the same category with
-            # group with previous results using the same category
+                # this one
-            # if the group can accept more result and is not too far
+                index = grp["index"]
-            # from the current position
-            if current is not None and (current['count'] > 0) and (len(gresults) - current['index'] < 20):
-                # group with the previous results using
-                # the same category with this one
-                index = current['index']
                 gresults.insert(index, res)
 
-                # update every index after the current one
+                # update every index after the current one (including the
-                # (including the current one)
+                # current one)
-                for k in categoryPositions:  # pylint: disable=consider-using-dict-items
+                for item in categoryPositions.values():
-                    v = categoryPositions[k]['index']
+                    v = item["index"]
                     if v >= index:
-                        categoryPositions[k]['index'] = v + 1
+                        item["index"] = v + 1
 
                 # update this category
-                current['count'] -= 1
+                grp["count"] -= 1
 
             else:
-                # same category
                 gresults.append(res)
-
                 # update categoryIndex
-                categoryPositions[category] = {'index': len(gresults), 'count': 8}
+                categoryPositions[category] = {"index": len(gresults), "count": max_count}
+                continue
 
-        # update _merged_results
+        self._main_results_sorted = gresults
-        self._merged_results = gresults
+        return self._main_results_sorted
-
-    def get_ordered_results(self):
-        if not self._closed:
-            self.close()
-        return self._merged_results
-
-    def results_length(self):
-        return len(self._merged_results)
 
     @property
     def number_of_results(self) -> int:
         """Returns the average of results number, returns zero if the average
         result number is smaller than the actual result count."""
 
-        with self._lock:
+        if not self._closed:
-            if not self._closed:
+            log.error("call to ResultContainer.number_of_results before ResultContainer.close")
-                logger.error("call to ResultContainer.number_of_results before ResultContainer.close")
+            return 0
-                return 0
 
+        with self._lock:
             resultnum_sum = sum(self._number_of_results)
             if not resultnum_sum or not self._number_of_results:
                 return 0
 
             average = int(resultnum_sum / len(self._number_of_results))
-            if average < self.results_length():
+            if average < len(self.get_ordered_results()):
                 average = 0
             return average
 
     def add_unresponsive_engine(self, engine_name: str, error_type: str, suspended: bool = False):
         with self._lock:
             if self._closed:
-                logger.error("call to ResultContainer.add_unresponsive_engine after ResultContainer.close")
+                log.error("call to ResultContainer.add_unresponsive_engine after ResultContainer.close")
                 return
-            if engines[engine_name].display_error_messages:
+            if searx.engines.engines[engine_name].display_error_messages:
                 self.unresponsive_engines.add(UnresponsiveEngine(engine_name, error_type, suspended))
 
     def add_timing(self, engine_name: str, engine_time: float, page_load_time: float):
         with self._lock:
             if self._closed:
-                logger.error("call to ResultContainer.add_timing after ResultContainer.close")
+                log.error("call to ResultContainer.add_timing after ResultContainer.close")
                 return
             self.timings.append(Timing(engine_name, total=engine_time, load=page_load_time))
 
     def get_timings(self):
         with self._lock:
             if not self._closed:
-                logger.error("call to ResultContainer.get_timings before ResultContainer.close")
+                log.error("call to ResultContainer.get_timings before ResultContainer.close")
                 return []
             return self.timings
+
+
+def merge_two_infoboxes(origin: LegacyResult, other: LegacyResult):
+    """Merges the values from ``other`` into ``origin``."""
+    # pylint: disable=too-many-branches
+    weight1 = getattr(searx.engines.engines[origin.engine], "weight", 1)
+    weight2 = getattr(searx.engines.engines[other.engine], "weight", 1)
+
+    if weight2 > weight1:
+        origin.engine = other.engine
+
+    origin.engines |= other.engines
+
+    if other.urls:
+        url_items = origin.get("urls", [])
+
+        for url2 in other.urls:
+            unique_url = True
+            entity_url2 = url2.get("entity")
+
+            for url1 in origin.get("urls", []):
+                if (entity_url2 is not None and entity_url2 == url1.get("entity")) or (
+                    url1.get("url") == url2.get("url")
+                ):
+                    unique_url = False
+                    break
+            if unique_url:
+                url_items.append(url2)
+
+        origin.urls = url_items
+
+    if other.img_src:
+        if not origin.img_src:
+            origin.img_src = other.img_src
+        elif weight2 > weight1:
+            origin.img_src = other.img_src
+
+    if other.attributes:
+        if not origin.attributes:
+            origin.attributes = other.attributes
+        else:
+            attr_names_1 = set()
+            for attr in origin.attributes:
+                label = attr.get("label")
+                if label:
+                    attr_names_1.add(label)
+
+                entity = attr.get("entity")
+                if entity:
+                    attr_names_1.add(entity)
+
+            for attr in other.attributes:
+                if attr.get("label") not in attr_names_1 and attr.get('entity') not in attr_names_1:
+                    origin.attributes.append(attr)
+
+    if other.content:
+        if not origin.content:
+            origin.content = other.content
+        elif len(other.content) > len(origin.content):
+            origin.content = other.content
+
+
+def merge_two_main_results(origin: MainResult | LegacyResult, other: MainResult | LegacyResult):
+    """Merges the values from ``other`` into ``origin``."""
+
+    if len(other.content) > len(origin.content):
+        # use content with more text
+        origin.content = other.content
+
+    # use title with more text
+    if len(other.title) > len(origin.title):
+        origin.title = other.title
+
+    # merge all result's parameters not found in origin
+    if isinstance(other, MainResult) and isinstance(origin, MainResult):
+        origin.defaults_from(other)
+    elif isinstance(other, LegacyResult) and isinstance(origin, LegacyResult):
+        origin.defaults_from(other)
+
+    # add engine to list of result-engines
+    origin.engines.add(other.engine or "")
+
+    # use https, ftps, .. if possible
+    if origin.parsed_url and not origin.parsed_url.scheme.endswith("s"):
+        if other.parsed_url and other.parsed_url.scheme.endswith("s"):
+            origin.parsed_url = origin.parsed_url._replace(scheme=other.parsed_url.scheme)
+            origin.url = origin.parsed_url.geturl()

searx/settings.yml

@@ -1888,15 +1888,15 @@ engines:
   # For this demo of the sqlite engine download:
   #   https://liste.mediathekview.de/filmliste-v2.db.bz2
   # and unpack into searx/data/filmliste-v2.db
-  # Query to test: "!demo concert"
+  # Query to test: "!mediathekview concert"
   #
-  # - name: demo
+  # - name: mediathekview
   #   engine: sqlite
-  #   shortcut: demo
+  #   shortcut: mediathekview
-  #   categories: general
+  #   categories: [general, videos]
-  #   result_template: default.html
+  #   result_type: MainResult
   #   database: searx/data/filmliste-v2.db
-  #   query_str:  >-
+  #   query_str: >-
   #     SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,
   #            COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,
   #            description AS content

searx/templates/simple/result_templates/key-value.html

@@ -1,11 +0,0 @@
-<table>
-    {% for key, value in result.items() %}
-    {% if key in ['engine', 'engines', 'template', 'score', 'category', 'positions', 'parsed_url'] %}
-        {% continue %}
-    {% endif %}
-    <tr>
-        <td><b>{{ key|upper }}</b>: {{ value }}</td>
-    </tr>
-    {% endfor %}
-</table>
-<div class="engines">{% for engine in result.engines %}<span>{{ engine }}</span>{% endfor %}</div>{{- '' -}}

searx/templates/simple/result_templates/keyvalue.html

@@ -0,0 +1,21 @@
+<article class="result result-keyvalue {% if result.category -%}category-{{ result.category }}{%- endif -%}">
+  <table>
+    {%- if result.caption %}<caption>{{ result.caption }}</caption>{%- endif -%}
+    {%- if result.key_title or result.value_title %}
+      <thead>
+        <tr>
+          <th class="col-key" scope="col" >{{result.key_title}}</th>
+          <th class="col-value" scope="col" >{{result.value_title}}</th>
+        </tr>
+      </thead>
+    {%- endif -%}
+    {%- for key, value in result.kvmap.items() -%}
+      <tr class="{{ loop.cycle('odd', 'even') }}">
+        <th class="col-key" scope="row">{{ key }}</th>{{- '' -}}
+        <td class="col-value">{{ value }}</td>{{- '' -}}
+      </tr>
+    {%- endfor -%}
+  </table>{{- '' -}}
+  <div class="engines">{% for engine in result.engines %}<span>{{ engine }}</span>{% endfor %}</div>{{- '' -}}
+  <div class="break"></div>{{- '' -}}
+</article>

searx/translations/bg/LC_MESSAGES/messages.po

@@ -15,13 +15,14 @@
 # stoychevww <stoychevww@users.noreply.translate.codeberg.org>, 2024, 2025.
 # Anonymous <anonymous@users.noreply.translate.codeberg.org>, 2025.
 # thenack0 <thenack0@users.noreply.translate.codeberg.org>, 2025.
+# return42 <return42@noreply.codeberg.org>, 2025.
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2025-01-29 05:08+0000\n"
-"PO-Revision-Date: 2025-02-15 15:34+0000\n"
+"PO-Revision-Date: 2025-03-14 07:09+0000\n"
-"Last-Translator: thenack0 <thenack0@users.noreply.translate.codeberg.org>\n"
+"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
 "Language-Team: Bulgarian <https://translate.codeberg.org/projects/searxng/"
 "searxng/bg/>\n"
 "Language: bg\n"
@@ -29,7 +30,7 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
-"X-Generator: Weblate 5.9.2\n"
+"X-Generator: Weblate 5.10.2\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -60,7 +61,7 @@ msgstr "музика"
 #. CATEGORY_NAMES['SOCIAL_MEDIA']
 #: searx/searxng.msg
 msgid "social media"
-msgstr "социална медия"
+msgstr "социална мрежа"
 
 #. CATEGORY_NAMES['IMAGES']
 #: searx/searxng.msg
@@ -448,9 +449,8 @@ msgid "Compute {func} of the arguments"
 msgstr ""
 
 #: searx/engines/openstreetmap.py:158
-#, fuzzy
 msgid "Show route in map .."
-msgstr "Покажи маршрута в картата..."
+msgstr "Покажи маршрута в картата.."
 
 #: searx/engines/pdbe.py:96
 msgid "{title} (OBSOLETE)"
@@ -563,13 +563,12 @@ msgid "Self Information"
 msgstr "Лична информация"
 
 #: searx/plugins/self_info.py:38
-#, fuzzy
 msgid ""
 "Displays your IP if the query is \"ip\" and your user agent if the query "
 "is \"user-agent\"."
 msgstr ""
 "Показва твоят IP адрес и твоят потребителски агент, ако заявката е "
-"\"потребителски агент\""
+"\"потребителски агент\"."
 
 #: searx/plugins/self_info.py:52
 msgid "Your IP is: "
@@ -577,7 +576,7 @@ msgstr "Твоето IP е: "
 
 #: searx/plugins/self_info.py:55
 msgid "Your user-agent is: "
-msgstr ""
+msgstr "Вашият потребителски агент е: "
 
 #: searx/plugins/tor_check.py:29
 msgid "Tor check plugin"
@@ -593,15 +592,15 @@ msgstr ""
 
 #: searx/plugins/tor_check.py:69
 msgid "Could not download the list of Tor exit-nodes from"
-msgstr ""
+msgstr "Не може да се изтегли списъкът с изходни възли на Tor от"
 
 #: searx/plugins/tor_check.py:81
 msgid "You are using Tor and it looks like you have the external IP address"
-msgstr ""
+msgstr "Използвате Tor и изглежда, че имате външен IP адрес"
 
 #: searx/plugins/tor_check.py:85
 msgid "You are not using Tor and you have the external IP address"
-msgstr ""
+msgstr "Не използвате Tor и имате външен IP адрес"
 
 #: searx/plugins/tracker_url_remover.py:18
 msgid "Tracker URL remover"
@@ -680,7 +679,7 @@ msgstr "Дължина"
 
 #: searx/templates/simple/macros.html:41
 msgid "Views"
-msgstr ""
+msgstr "Изгледи"
 
 #: searx/templates/simple/macros.html:42
 #: searx/templates/simple/result_templates/files.html:34
@@ -929,7 +928,7 @@ msgstr "Примери"
 
 #: searx/templates/simple/answer/translations.html:21
 msgid "Definitions"
-msgstr ""
+msgstr "Дефиниции"
 
 #: searx/templates/simple/answer/translations.html:30
 msgid "Synonyms"
@@ -953,7 +952,7 @@ msgstr "Съобщения от търсачките"
 
 #: searx/templates/simple/elements/engines_msg.html:7
 msgid "seconds"
-msgstr ""
+msgstr "секунди"
 
 #: searx/templates/simple/elements/search_url.html:3
 msgid "Search URL"
@@ -1090,7 +1089,7 @@ msgstr "Позволи"
 
 #: searx/templates/simple/preferences/answerers.html:5
 msgid "Keywords (first word in query)"
-msgstr ""
+msgstr "Ключови думи (първата дума в заявката)"
 
 #: searx/templates/simple/preferences/answerers.html:6
 #: searx/templates/simple/result_templates/packages.html:7
@@ -1184,7 +1183,7 @@ msgstr "Хеш на предпочитанията"
 
 #: searx/templates/simple/preferences/doi_resolver.html:1
 msgid "Digital Object Identifier (DOI)"
-msgstr ""
+msgstr "Идентификатор на цифров обект (DOI)"
 
 #: searx/templates/simple/preferences/doi_resolver.html:6
 msgid "Open Access DOI resolver"

searx/translations/pl/LC_MESSAGES/messages.po

@@ -27,14 +27,14 @@
 # mradalbert <mradalbert@users.noreply.translate.codeberg.org>, 2025.
 # Bubowny <bubowny@users.noreply.translate.codeberg.org>, 2025.
 # matsob0123 <matsob0123@users.noreply.translate.codeberg.org>, 2025.
+# return42 <return42@noreply.codeberg.org>, 2025.
 msgid ""
 msgstr ""
 "Project-Id-Version: searx\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
 "POT-Creation-Date: 2025-01-29 05:08+0000\n"
-"PO-Revision-Date: 2025-02-02 19:48+0000\n"
+"PO-Revision-Date: 2025-03-14 07:09+0000\n"
-"Last-Translator: matsob0123 <matsob0123@users.noreply.translate.codeberg.org>"
+"Last-Translator: return42 <return42@noreply.codeberg.org>\n"
-"\n"
 "Language-Team: Polish <https://translate.codeberg.org/projects/searxng/"
 "searxng/pl/>\n"
 "Language: pl\n"
@@ -44,7 +44,7 @@ msgstr ""
 "Plural-Forms: nplurals=4; plural=(n==1 ? 0 : (n%10>=2 && n%10<=4) && ("
 "n%100<12 || n%100>14) ? 1 : n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && "
 "n%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3);\n"
-"X-Generator: Weblate 5.9.2\n"
+"X-Generator: Weblate 5.10.2\n"
 "Generated-By: Babel 2.16.0\n"
 
 #. CONSTANT_NAMES['NO_SUBGROUPING']
@@ -799,7 +799,7 @@ msgstr "Obecnie używane wyszukiwarki"
 
 #: searx/templates/simple/preferences.html:243
 msgid "Special Queries"
-msgstr "Specialne Zapytania"
+msgstr "Specjalne Zapytania"
 
 #: searx/templates/simple/preferences.html:251
 msgid "Cookies"

tests/unit/engines/test_command.py

@@ -2,6 +2,8 @@
 # pylint: disable=missing-module-docstring,disable=missing-class-docstring,invalid-name
 
 from searx.engines import command as command_engine
+from searx.result_types import KeyValue
+
 from tests import SearxTestCase
 
 
@@ -12,14 +14,15 @@ class TestCommandEngine(SearxTestCase):
         ls_engine.command = ['seq', '{{QUERY}}']
         ls_engine.delimiter = {'chars': ' ', 'keys': ['number']}
         expected_results = [
-            {'number': '1', 'template': 'key-value.html'},
+            KeyValue(kvmap={'number': 1}),
-            {'number': '2', 'template': 'key-value.html'},
+            KeyValue(kvmap={'number': 2}),
-            {'number': '3', 'template': 'key-value.html'},
+            KeyValue(kvmap={'number': 3}),
-            {'number': '4', 'template': 'key-value.html'},
+            KeyValue(kvmap={'number': 4}),
-            {'number': '5', 'template': 'key-value.html'},
+            KeyValue(kvmap={'number': 5}),
         ]
         results = ls_engine.search('5', {'pageno': 1})
-        self.assertEqual(results, expected_results)
+        for i, expected in enumerate(expected_results):
+            self.assertEqual(results[i].kvmap["number"], str(expected.kvmap["number"]))
 
     def test_delimiter_parsing(self):
         searx_logs = '''DEBUG:searx.webapp:static directory is /home/n/p/searx/searx/static
@@ -39,94 +42,85 @@ INFO:werkzeug: * Debugger PIN: 299-578-362'''
         echo_engine.command = ['echo', searx_logs]
         echo_engine.delimiter = {'chars': ':', 'keys': ['level', 'component', 'message']}
 
-        expected_results_by_page = [
+        page1 = [
-            [
+            {
-                {
+                'component': 'searx.webapp',
-                    'component': 'searx.webapp',
+                'message': 'static directory is /home/n/p/searx/searx/static',
-                    'message': 'static directory is /home/n/p/searx/searx/static',
+                'level': 'DEBUG',
-                    'template': 'key-value.html',
+            },
-                    'level': 'DEBUG',
+            {
-                },
+                'component': 'searx.webapp',
-                {
+                'message': 'templates directory is /home/n/p/searx/searx/templates',
-                    'component': 'searx.webapp',
+                'level': 'DEBUG',
-                    'message': 'templates directory is /home/n/p/searx/searx/templates',
+            },
-                    'template': 'key-value.html',
+            {
-                    'level': 'DEBUG',
+                'component': 'searx.engines',
-                },
+                'message': 'soundcloud engine: Starting background initialization',
-                {
+                'level': 'DEBUG',
-                    'component': 'searx.engines',
+            },
-                    'message': 'soundcloud engine: Starting background initialization',
+            {
-                    'template': 'key-value.html',
+                'component': 'searx.engines',
-                    'level': 'DEBUG',
+                'message': 'wolframalpha engine: Starting background initialization',
-                },
+                'level': 'DEBUG',
-                {
+            },
-                    'component': 'searx.engines',
+            {
-                    'message': 'wolframalpha engine: Starting background initialization',
+                'component': 'searx.engines',
-                    'template': 'key-value.html',
+                'message': 'locate engine: Starting background initialization',
-                    'level': 'DEBUG',
+                'level': 'DEBUG',
-                },
+            },
-                {
+            {
-                    'component': 'searx.engines',
+                'component': 'searx.engines',
-                    'message': 'locate engine: Starting background initialization',
+                'message': 'regex search in files engine: Starting background initialization',
-                    'template': 'key-value.html',
+                'level': 'DEBUG',
-                    'level': 'DEBUG',
+            },
-                },
+            {
-                {
+                'component': 'urllib3.connectionpool',
-                    'component': 'searx.engines',
+                'message': 'Starting new HTTPS connection (1): www.wolframalpha.com',
-                    'message': 'regex search in files engine: Starting background initialization',
+                'level': 'DEBUG',
-                    'template': 'key-value.html',
+            },
-                    'level': 'DEBUG',
+            {
-                },
+                'component': 'urllib3.connectionpool',
-                {
+                'message': 'Starting new HTTPS connection (1): soundcloud.com',
-                    'component': 'urllib3.connectionpool',
+                'level': 'DEBUG',
-                    'message': 'Starting new HTTPS connection (1): www.wolframalpha.com',
+            },
-                    'template': 'key-value.html',
+            {
-                    'level': 'DEBUG',
+                'component': 'searx.engines',
-                },
+                'message': 'find engine: Starting background initialization',
-                {
+                'level': 'DEBUG',
-                    'component': 'urllib3.connectionpool',
+            },
-                    'message': 'Starting new HTTPS connection (1): soundcloud.com',
+            {
-                    'template': 'key-value.html',
+                'component': 'searx.engines',
-                    'level': 'DEBUG',
+                'message': 'pattern search in files engine: Starting background initialization',
-                },
+                'level': 'DEBUG',
-                {
+            },
-                    'component': 'searx.engines',
+        ]
-                    'message': 'find engine: Starting background initialization',
+        page2 = [
-                    'template': 'key-value.html',
+            {
-                    'level': 'DEBUG',
+                'component': 'searx.webapp',
-                },
+                'message': 'starting webserver on 127.0.0.1:8888',
-                {
+                'level': 'DEBUG',
-                    'component': 'searx.engines',
+            },
-                    'message': 'pattern search in files engine: Starting background initialization',
+            {
-                    'template': 'key-value.html',
+                'component': 'werkzeug',
-                    'level': 'DEBUG',
+                'message': ' * Debugger is active!',
-                },
+                'level': 'WARNING',
-            ],
+            },
-            [
+            {
-                {
+                'component': 'werkzeug',
-                    'component': 'searx.webapp',
+                'message': ' * Debugger PIN: 299-578-362',
-                    'message': 'starting webserver on 127.0.0.1:8888',
+                'level': 'INFO',
-                    'template': 'key-value.html',
+            },
-                    'level': 'DEBUG',
-                },
-                {
-                    'component': 'werkzeug',
-                    'message': ' * Debugger is active!',
-                    'template': 'key-value.html',
-                    'level': 'WARNING',
-                },
-                {
-                    'component': 'werkzeug',
-                    'message': ' * Debugger PIN: 299-578-362',
-                    'template': 'key-value.html',
-                    'level': 'INFO',
-                },
-            ],
         ]
 
+        page1 = [KeyValue(kvmap=row) for row in page1]
+        page2 = [KeyValue(kvmap=row) for row in page2]
+
+        expected_results_by_page = [page1, page2]
         for i in [0, 1]:
             results = echo_engine.search('', {'pageno': i + 1})
-            self.assertEqual(results, expected_results_by_page[i])
+            page = expected_results_by_page[i]
+            for i, expected in enumerate(page):
+                self.assertEqual(expected.kvmap["message"], str(results[i].kvmap["message"]))
 
     def test_regex_parsing(self):
         txt = '''commit 35f9a8c81d162a361b826bbcd4a1081a4fbe76a7
@@ -165,26 +159,25 @@ commit '''
                 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
                 'date': 'Date:   Tue Oct 15 11:31:33 2019 +0200',
                 'message': '\n\nfirst interesting message',
-                'template': 'key-value.html',
             },
             {
                 'commit': '6c3c206316153ccc422755512bceaa9ab0b14faa',
                 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
                 'date': 'Date:   Mon Oct 14 17:10:08 2019 +0200',
                 'message': '\n\nsecond interesting message',
-                'template': 'key-value.html',
             },
             {
                 'commit': 'd8594d2689b4d5e0d2f80250223886c3a1805ef5',
                 'author': ' Noémi Ványi <sitbackandwait@gmail.com>',
                 'date': 'Date:   Mon Oct 14 14:45:05 2019 +0200',
                 'message': '\n\nthird interesting message',
-                'template': 'key-value.html',
             },
         ]
 
+        expected_results = [KeyValue(kvmap=kvmap) for kvmap in expected_results]
         results = git_log_engine.search('', {'pageno': 1})
-        self.assertEqual(results, expected_results)
+        for i, expected in enumerate(expected_results):
+            self.assertEqual(expected.kvmap["message"], str(results[i].kvmap["message"]))
 
     def test_working_dir_path_query(self):
         ls_engine = command_engine

tests/unit/test_engine_mariadb_server.py

@@ -1,30 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-or-later
-# pylint: disable=missing-module-docstring,disable=missing-class-docstring,invalid-name
-
-from unittest.mock import MagicMock, Mock
-
-from searx.engines import mariadb_server
-from tests import SearxTestCase
-
-
-class MariadbServerTests(SearxTestCase):
-
-    def test_init_no_query_str_raises(self):
-        self.assertRaises(ValueError, lambda: mariadb_server.init({}))
-
-    def test_init_non_select_raises(self):
-        self.assertRaises(ValueError, lambda: mariadb_server.init({'query_str': 'foobar'}))
-
-    def test_search_returns_results(self):
-        test_string = 'FOOBAR'
-        cursor_mock = MagicMock()
-        with cursor_mock as setup:  # pylint: disable=not-context-manager
-            setup.__iter__ = Mock(return_value=iter([{test_string, 1}]))
-            setup.description = [[test_string]]
-        conn_mock = Mock()
-        conn_mock.cursor.return_value = cursor_mock
-        mariadb_server._connection = conn_mock  # pylint: disable=protected-access
-        results = mariadb_server.search(test_string, {'pageno': 1})
-        self.assertEqual(1, len(results))
-        self.assertIn(test_string, results[0])
-        self.assertEqual(mariadb_server.result_template, results[0]['template'])

tests/unit/test_results.py

@@ -23,8 +23,11 @@ class ResultContainerTestCase(SearxTestCase):
         container.extend("google", [result])
         container.close()
 
-        self.assertEqual(container.results_length(), 1)
+        self.assertEqual(len(container.get_ordered_results()), 1)
-        self.assertIn(LegacyResult(result), container.get_ordered_results())
+
+        res = LegacyResult(result)
+        res.normalize_result_fields()
+        self.assertIn(res, container.get_ordered_results())
 
     def test_one_suggestion(self):
         result = dict(suggestion="lorem ipsum ..")
@@ -33,7 +36,7 @@ class ResultContainerTestCase(SearxTestCase):
         container.extend("duckduckgo", [result])
         container.close()
 
-        self.assertEqual(container.results_length(), 0)
+        self.assertEqual(len(container.get_ordered_results()), 0)
         self.assertEqual(len(container.suggestions), 1)
         self.assertIn(result["suggestion"], container.suggestions)
 
@@ -42,6 +45,7 @@ class ResultContainerTestCase(SearxTestCase):
         result = LegacyResult(
             url="https://example.org", title="very long title, lorem ipsum", content="Lorem ipsum dolor sit amet .."
         )
+        result.normalize_result_fields()
         eng1 = dict(url=result.url, title="short title", content=result.content, engine="google")
         eng2 = dict(url="http://example.org", title=result.title, content="lorem ipsum", engine="duckduckgo")
 
@@ -50,7 +54,7 @@ class ResultContainerTestCase(SearxTestCase):
         container.close()
 
         result_list = container.get_ordered_results()
-        self.assertEqual(container.results_length(), 1)
+        self.assertEqual(len(container.get_ordered_results()), 1)
         self.assertIn(result, result_list)
         self.assertEqual(result_list[0].title, result.title)
         self.assertEqual(result_list[0].content, result.content)

tests/unit/test_webapp.py

@@ -148,11 +148,10 @@ class ViewsTestCase(SearxTestCase):  # pylint: disable=too-many-public-methods
 
     def test_search_csv(self):
         result = self.client.post('/search', data={'q': 'test', 'format': 'csv'})
-
         self.assertEqual(
             b'title,url,content,host,engine,score,type\r\n'
-            b'First Test,http://first.test.xyz,first test content,first.test.xyz,startpage,,result\r\n'  # noqa
+            + b'First Test,http://first.test.xyz,first test content,first.test.xyz,startpage,0,result\r\n'
-            b'Second Test,http://second.test.xyz,second test content,second.test.xyz,youtube,,result\r\n',  # noqa
+            + b'Second Test,http://second.test.xyz,second test content,second.test.xyz,youtube,0,result\r\n',
             result.data,
         )
 

utils/lib_sxng_static.sh

@@ -5,6 +5,7 @@
 STATIC_BUILD_COMMIT="[build] /static"
 STATIC_BUILT_PATHS=(
     'searx/static/themes/simple'
+    'client/simple/package-lock.json'
 )
 
 static.help(){