[mod] slightly improve make node.clean themes.all

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

.nvmrc

@@ -1 +1 @@
-v20.10
+v23.5

.tool-versions

@@ -1,2 +1,4 @@
-python 3.12.0
-shellcheck 0.9.0
+nodejs 23.5.0
+python 3.13.1
+shellcheck 0.10.0
+sqlite 3.47.2

docs/dev/engines/index.rst

@@ -45,6 +45,7 @@ Online Engines
    demo/demo_online
    xpath
    mediawiki
+   json_engine
 
 .. toctree::
    :maxdepth: 1

docs/dev/engines/json_engine.rst

@@ -0,0 +1,13 @@
+.. _json_engine engine:
+
+============
+JSON Engine
+============
+
+.. contents::
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.engines.json_engine
+  :members:

docs/dev/quickstart.rst

@@ -39,10 +39,8 @@ to our ":ref:`how to contribute`" guideline.
    - :ref:`make themes`
 
 If you implement themes, you will need to setup a :ref:`Node.js environment
-<make node.env>`: ``make node.env``
-
-Before you call *make run* (2.), you need to compile the modified styles and
-JavaScript: ``make themes.all``
+<make node.env>`.  Before you call *make run* (2.), you need to compile the
+modified styles and JavaScript: ``make node.clean themes.all``
 
 Alternatively you can also compile selective the theme you have modified,
 e.g. the *simple* theme.

manage

@@ -94,8 +94,6 @@ pyenv.:
   OK        : test if virtualenv is OK
 format.:
   python    : format Python code source using black
-pygments.:
-  less      : build LESS files for pygments
 EOF
     go.help
     node.help
@@ -253,15 +251,6 @@ gecko.driver() {
     dump_return $?
 }
 
-pygments.less() {
-    build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
-    if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
-        build_msg PYGMENTS "building LESS files for pygments failed"
-        return 1
-    fi
-    return 0
-}
-
 py.build() {
     build_msg BUILD "python package ${PYDIST}"
     pyenv.cmd python setup.py \

package.json

@@ -1,7 +1,7 @@
 {
   "dependencies": {
-    "eslint": "^9.0.0",
-    "pyright": "^1.1.329"
+    "eslint": "^9.17.0",
+    "pyright": "^1.1.391"
   },
   "scripts": {
     "clean": "rm -Rf node_modules package-lock.json"

searx/engines/json_engine.py

@@ -2,12 +2,74 @@
 """The JSON engine is a *generic* engine with which it is possible to configure
 engines in the settings.
 
-.. todo::
+Configuration
+=============
 
-   - The JSON engine needs documentation!!
+Request:
 
-   - The parameters of the JSON engine should be adapted to those of the XPath
-     engine.
+- :py:obj:`search_url`
+- :py:obj:`lang_all`
+- :py:obj:`soft_max_redirects`
+- :py:obj:`method`
+- :py:obj:`request_body`
+- :py:obj:`cookies`
+- :py:obj:`headers`
+
+Paging:
+
+- :py:obj:`paging`
+- :py:obj:`page_size`
+- :py:obj:`first_page_num`
+
+Time Range:
+
+- :py:obj:`time_range_support`
+- :py:obj:`time_range_url`
+- :py:obj:`time_range_map`
+
+Safe-Search:
+
+- :py:obj:`safe_search_support`
+- :py:obj:`safe_search_map`
+
+Response:
+
+- :py:obj:`title_html_to_text`
+- :py:obj:`content_html_to_text`
+- :py:obj:`no_result_for_http_status`
+
+JSON query:
+
+- :py:obj:`results_query`
+- :py:obj:`url_query`
+- :py:obj:`url_prefix`
+- :py:obj:`title_query`
+- :py:obj:`content_query`
+- :py:obj:`thumbnail_query`
+- :py:obj:`thumbnail_prefix`
+- :py:obj:`suggestion_query`
+
+
+Example
+=======
+
+Here is a simple example of a JSON engine configure in the :ref:`settings
+engine` section, further read :ref:`engines-dev`.
+
+.. code:: yaml
+
+  - name : mdn
+    engine : json_engine
+    paging : True
+    search_url : https://developer.mozilla.org/api/v1/search?q={query}&page={pageno}
+    results_query : documents
+    url_query : mdn_url
+    url_prefix : https://developer.mozilla.org
+    title_query : title
+    content_query : summary
+
+Implementations
+===============
 
 """
 
@@ -15,35 +77,180 @@ from collections.abc import Iterable
 from json import loads
 from urllib.parse import urlencode
 from searx.utils import to_string, html_to_text
+from searx.network import raise_for_httperror
 
-# parameters for generating a request
 search_url = None
+"""
+Search URL of the engine.  Example::
+
+    https://example.org/?search={query}&page={pageno}{time_range}{safe_search}
+
+Replacements are:
+
+``{query}``:
+  Search terms from user.
+
+``{pageno}``:
+  Page number if engine supports paging :py:obj:`paging`
+
+``{lang}``:
+  ISO 639-1 language code (en, de, fr ..)
+
+``{time_range}``:
+  :py:obj:`URL parameter <time_range_url>` if engine :py:obj:`supports time
+  range <time_range_support>`.  The value for the parameter is taken from
+  :py:obj:`time_range_map`.
+
+``{safe_search}``:
+  Safe-search :py:obj:`URL parameter <safe_search_map>` if engine
+  :py:obj:`supports safe-search <safe_search_support>`.  The ``{safe_search}``
+  replacement is taken from the :py:obj:`safes_search_map`.  Filter results::
+
+      0: none, 1: moderate, 2:strict
+
+  If not supported, the URL parameter is an empty string.
+
+"""
+
+lang_all = 'en'
+'''Replacement ``{lang}`` in :py:obj:`search_url` if language ``all`` is
+selected.
+'''
+
+no_result_for_http_status = []
+'''Return empty result for these HTTP status codes instead of throwing an error.
+
+.. code:: yaml
+
+    no_result_for_http_status: []
+'''
+
+soft_max_redirects = 0
+'''Maximum redirects, soft limit. Record an error but don't stop the engine'''
+
 method = 'GET'
+'''Some engines might require to do POST requests for search.'''
+
 request_body = ''
+'''The body of the request.  This can only be used if different :py:obj:`method`
+is set, e.g. ``POST``. For formatting see the documentation of :py:obj:`search_url`.
+
+Note: Curly brackets which aren't encapsulating a replacement placeholder
+must be escaped by doubling each ``{`` and ``}``.
+
+.. code:: yaml
+
+    request_body: >-
+      {{
+        "search": "{query}",
+        "page": {pageno},
+        "extra": {{
+          "time_range": {time_range},
+          "rating": "{safe_search}"
+        }}
+      }}
+'''
 
 cookies = {}
+'''Some engines might offer different result based on cookies.
+Possible use-case: To set safesearch cookie.'''
+
 headers = {}
 '''Some engines might offer different result based on cookies or headers.
 Possible use-case: To set safesearch cookie or header to moderate.'''
 
 paging = False
-# parameters for engines with paging support
-#
-# number of results on each page
-# (only needed if the site requires not a page number, but an offset)
-page_size = 1
-# number of the first page (usually 0 or 1)
-first_page_num = 1
+'''Engine supports paging [True or False].'''
+
+page_size = 1
+'''Number of results on each page.  Only needed if the site requires not a page
+number, but an offset.'''
+
+first_page_num = 1
+'''Number of the first page (usually 0 or 1).'''
 
-# parameters for parsing the response
 results_query = ''
+'''JSON query for the list of result items.
+
+The query string is a slash `/` separated path of JSON key names.
+Array entries can be specified using the index or can be omitted entirely,
+in which case each entry is considered -
+most implementations will default to the first entry in this case.
+'''
+
 url_query = None
+'''JSON query of result's ``url``. For the query string documentation see :py:obj:`results_query`'''
+
 url_prefix = ""
+'''String to prepend to the result's ``url``.'''
+
 title_query = None
+'''JSON query of result's ``title``. For the query string documentation see :py:obj:`results_query`'''
+
 content_query = None
+'''JSON query of result's ``content``. For the query string documentation see :py:obj:`results_query`'''
+
+thumbnail_query = False
+'''JSON query of result's ``thumbnail``. For the query string documentation see :py:obj:`results_query`'''
+
+thumbnail_prefix = ''
+'''String to prepend to the result's ``thumbnail``.'''
+
 suggestion_query = ''
+'''JSON query of result's ``suggestion``. For the query string documentation see :py:obj:`results_query`'''
+
 title_html_to_text = False
+'''Extract text from a HTML title string'''
+
 content_html_to_text = False
+'''Extract text from a HTML content string'''
+
+time_range_support = False
+'''Engine supports search time range.'''
+
+time_range_url = '&hours={time_range_val}'
+'''Time range URL parameter in the in :py:obj:`search_url`.  If no time range is
+requested by the user, the URL parameter is an empty string.  The
+``{time_range_val}`` replacement is taken from the :py:obj:`time_range_map`.
+
+.. code:: yaml
+
+    time_range_url : '&days={time_range_val}'
+'''
+
+time_range_map = {
+    'day': 24,
+    'week': 24 * 7,
+    'month': 24 * 30,
+    'year': 24 * 365,
+}
+'''Maps time range value from user to ``{time_range_val}`` in
+:py:obj:`time_range_url`.
+
+.. code:: yaml
+
+    time_range_map:
+      day: 1
+      week: 7
+      month: 30
+      year: 365
+'''
+
+safe_search_support = False
+'''Engine supports safe-search.'''
+
+safe_search_map = {0: '&filter=none', 1: '&filter=moderate', 2: '&filter=strict'}
+'''Maps safe-search value to ``{safe_search}`` in :py:obj:`search_url`.
+
+.. code:: yaml
+
+    safesearch: true
+    safes_search_map:
+      0: '&filter=none'
+      1: '&filter=moderate'
+      2: '&filter=strict'
+
+'''
 
 
 def iterate(iterable):
@@ -102,10 +309,27 @@ def query(data, query_string):
 
 
 def request(query, params):  # pylint: disable=redefined-outer-name
-    fp = {'query': urlencode({'q': query})[2:]}  # pylint: disable=invalid-name
+    '''Build request parameters (see :ref:`engine request`).'''
+    lang = lang_all
+    if params['language'] != 'all':
+        lang = params['language'][:2]
 
-    if paging and search_url.find('{pageno}') >= 0:
-        fp['pageno'] = (params['pageno'] - 1) * page_size + first_page_num
+    time_range = ''
+    if params.get('time_range'):
+        time_range_val = time_range_map.get(params.get('time_range'))
+        time_range = time_range_url.format(time_range_val=time_range_val)
+
+    safe_search = ''
+    if params['safesearch']:
+        safe_search = safe_search_map[params['safesearch']]
+
+    fp = {  # pylint: disable=invalid-name
+        'query': urlencode({'q': query})[2:],
+        'lang': lang,
+        'pageno': (params['pageno'] - 1) * page_size + first_page_num,
+        'time_range': time_range,
+        'safe_search': safe_search,
+    }
 
     params['cookies'].update(cookies)
     params['headers'].update(headers)
@@ -118,6 +342,9 @@ def request(query, params):  # pylint: disable=redefined-outer-name
         fp['query'] = query
         params['data'] = request_body.format(**fp)
 
+    params['soft_max_redirects'] = soft_max_redirects
+    params['raise_for_httperror'] = False
+
     return params
 
 
@@ -125,47 +352,69 @@ def identity(arg):
     return arg
 
 
-def response(resp):
-    results = []
-    json = loads(resp.text)
-
+def extract_response_info(result):
     title_filter = html_to_text if title_html_to_text else identity
     content_filter = html_to_text if content_html_to_text else identity
 
+    tmp_result = {}
+
+    try:
+        url = query(result, url_query)[0]
+        tmp_result['url'] = url_prefix + to_string(url)
+
+        title = query(result, title_query)[0]
+        tmp_result['title'] = title_filter(to_string(title))
+    except:  # pylint: disable=bare-except
+        return None
+
+    try:
+        content = query(result, content_query)[0]
+        tmp_result['content'] = content_filter(to_string(content))
+    except:  # pylint: disable=bare-except
+        tmp_result['content'] = ""
+
+    try:
+        if thumbnail_query:
+            thumbnail_query_result = query(result, thumbnail_query)[0]
+            tmp_result['thumbnail'] = thumbnail_prefix + to_string(thumbnail_query_result)
+    except:  # pylint: disable=bare-except
+        pass
+
+    return tmp_result
+
+
+def response(resp):
+    '''Scrap *results* from the response (see :ref:`engine results`).'''
+    results = []
+
+    if no_result_for_http_status and resp.status_code in no_result_for_http_status:
+        return results
+
+    raise_for_httperror(resp)
+
+    if not resp.text:
+        return results
+
+    json = loads(resp.text)
+    is_onion = 'onions' in categories
+
     if results_query:
         rs = query(json, results_query)  # pylint: disable=invalid-name
         if not rs:
             return results
-        for result in rs[0]:
-            try:
-                url = query(result, url_query)[0]
-                title = query(result, title_query)[0]
-            except:  # pylint: disable=bare-except
-                continue
-            try:
-                content = query(result, content_query)[0]
-            except:  # pylint: disable=bare-except
-                content = ""
-            results.append(
-                {
-                    'url': url_prefix + to_string(url),
-                    'title': title_filter(to_string(title)),
-                    'content': content_filter(to_string(content)),
-                }
-            )
+        rs = rs[0]  # pylint: disable=invalid-name
     else:
-        for result in json:
-            url = query(result, url_query)[0]
-            title = query(result, title_query)[0]
-            content = query(result, content_query)[0]
+        rs = json  # pylint: disable=invalid-name
 
-            results.append(
-                {
-                    'url': url_prefix + to_string(url),
-                    'title': title_filter(to_string(title)),
-                    'content': content_filter(to_string(content)),
-                }
-            )
+    for result in rs:
+        tmp_result = extract_response_info(result)
+        if not tmp_result:
+            continue
+
+        if is_onion:
+            tmp_result['is_onion'] = True
+
+        results.append(tmp_result)
 
     if not suggestion_query:
         return results

searx/static/themes/simple/.eslintrc.json

@@ -1,28 +0,0 @@
-{
-    "env": {
-        "browser": true,
-        "es2021": true,
-        "node": true
-    },
-    "extends": "eslint:recommended",
-    "parserOptions": {
-        "ecmaVersion": 12
-    },
-    "rules": {
-        "indent": ["error", 2],
-        "keyword-spacing": ["error", { "before": true, "after": true }],
-        "no-trailing-spaces": 2,
-        "space-before-function-paren": ["error", "always"],
-        "space-infix-ops": "error",
-        "comma-spacing": ["error", { "before": false, "after": true }],
-        "brace-style": ["error", "1tbs", { "allowSingleLine": true }],
-        "curly": ["error", "multi-line"],
-        "block-spacing": ["error", "always"],
-        "dot-location": ["error", "property"],
-        "key-spacing": ["error", { "beforeColon": false, "afterColon": true }],
-        "spaced-comment": ["error", "always", {
-            "line": { "markers": ["*package", "!", "/", ",", "="] },
-            "block": { "balanced": true, "markers": ["*package", "!", ",", ":", "::", "flow-include"], "exceptions": ["*"] }
-        }]
-    }
-}

searx/static/themes/simple/.gitignore

@@ -1,2 +1 @@
 /node_modules
-package-lock.json

searx/static/themes/simple/.stylelintrc.json

@@ -1,7 +1,14 @@
 {
-  "extends": "stylelint-config-standard",
+  "extends": "stylelint-config-standard-less",
   "rules": {
     "declaration-empty-line-before": null,
-    "no-invalid-position-at-import-rule": null
+    "no-invalid-position-at-import-rule": null,
+    "property-no-vendor-prefix": null,
+    "selector-no-vendor-prefix": null,
+    "selector-attribute-quotes": null,
+    "shorthand-property-no-redundant-values": null,
+    "at-rule-no-vendor-prefix": null,
+    "selector-id-pattern": null,
+    "selector-class-pattern": null
   }
 }

searx/static/themes/simple/eslint.config.mjs

@@ -0,0 +1,68 @@
+import globals from "globals";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import js from "@eslint/js";
+import { FlatCompat } from "@eslint/eslintrc";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const compat = new FlatCompat({
+    baseDirectory: __dirname,
+    recommendedConfig: js.configs.recommended,
+    allConfig: js.configs.all
+});
+
+export default [...compat.extends("eslint:recommended"), {
+    languageOptions: {
+        globals: {
+            ...globals.browser,
+            ...globals.node,
+        },
+
+        ecmaVersion: 12,
+        sourceType: "commonjs",
+    },
+
+    rules: {
+        indent: ["error", 2],
+
+        "keyword-spacing": ["error", {
+            before: true,
+            after: true,
+        }],
+
+        "no-trailing-spaces": 2,
+        "space-before-function-paren": ["error", "always"],
+        "space-infix-ops": "error",
+
+        "comma-spacing": ["error", {
+            before: false,
+            after: true,
+        }],
+
+        "brace-style": ["error", "1tbs", {
+            allowSingleLine: true,
+        }],
+
+        curly: ["error", "multi-line"],
+        "block-spacing": ["error", "always"],
+        "dot-location": ["error", "property"],
+
+        "key-spacing": ["error", {
+            beforeColon: false,
+            afterColon: true,
+        }],
+
+        "spaced-comment": ["error", "always", {
+            line: {
+                markers: ["*package", "!", "/", ",", "="],
+            },
+
+            block: {
+                balanced: true,
+                markers: ["*package", "!", ",", ":", "::", "flow-include"],
+                exceptions: ["*"],
+            },
+        }],
+    },
+}];

searx/static/themes/simple/gruntfile.js

@@ -35,7 +35,7 @@ module.exports = function (grunt) {
     },
     eslint: {
       options: {
-        overrideConfigFile: '.eslintrc.json',
+        overrideConfigFile: 'eslint.config.mjs',
         failOnError: true,
         fix: grunt.option('fix')
       },

searx/static/themes/simple/package.json

@@ -1,28 +1,29 @@
 {
   "devDependencies": {
-    "eslint": "^9.0.0",
-    "grunt": "~1.6.1",
+    "ejs": "^3.1.10",
+    "eslint": "^9.17.0",
+    "grunt": "^1.6.1",
     "grunt-contrib-copy": "^1.0.0",
     "grunt-contrib-cssmin": "^5.0.0",
-    "grunt-contrib-less": "~3.0.0",
-    "grunt-contrib-uglify": "~5.2.1",
-    "grunt-xmlmin": "~0.1.8",
-    "grunt-contrib-watch": "~1.1.0",
-    "grunt-eslint": "^24.0.0",
-    "grunt-stylelint": "^0.16.0",
+    "grunt-contrib-less": "^3.0.0",
+    "grunt-contrib-uglify": "^5.2.2",
+    "grunt-contrib-watch": "^1.1.0",
+    "grunt-eslint": "^25.0.0",
     "grunt-image": "^6.4.0",
-    "ionicons": "^7.1.0",
-    "less": "^4.1.3",
-    "less-plugin-clean-css": "^1.5.1",
-    "sharp": "^0.33.0",
-    "stylelint": "^13.13.1",
-    "stylelint-config-standard": "^22.0.0",
-    "ejs": "^3.1.8",
-    "svgo": "^3.0.0"
+    "grunt-stylelint": "^0.20.1",
+    "grunt-xmlmin": "^0.1.8",
+    "ionicons": "^7.4.0",
+    "less": "^4.2.1",
+    "less-plugin-clean-css": "^1.6.0",
+    "sharp": "^0.33.5",
+    "stylelint": "^16.12.0",
+    "stylelint-config-standard": "^36.0.0",
+    "stylelint-config-standard-less": "^3.0.1",
+    "svgo": "^3.3.2"
   },
   "dependencies": {
-    "autocomplete-js": "2.7.1",
-    "leaflet": "^1.8.0",
+    "autocomplete-js": "^2.7.1",
+    "leaflet": "^1.9.4",
     "normalize.css": "^8.0.1"
   },
   "scripts": {
@@ -32,7 +33,7 @@
     "eslint": "grunt eslint",
     "eslint-fix": "grunt eslint --fix",
     "watch": "grunt watch",
-    "clean": "rm -Rf node_modules package-lock.json",
+    "clean": "rm -Rf node_modules",
     "stylelint": "grunt stylelint",
     "stylelint-fix": "grunt stylelint --fix"
   }

searx/static/themes/simple/src/less/definitions.less

@@ -85,8 +85,8 @@
   --color-result-detail-background: #242424;
   --color-result-detail-hr: #555;
   --color-result-detail-link: #8af;
-  --color-result-detail-loader-border: rgba(255, 255, 255, 0.2);
-  --color-result-detail-loader-borderleft: rgba(0, 0, 0, 0);
+  --color-result-detail-loader-border: rgb(255 255 255 / 20%);
+  --color-result-detail-loader-borderleft: rgb(0 0 0 / 0%);
   /// Toolkit Colors
   --color-toolkit-badge-font: #fff;
   --color-toolkit-badge-background: #545454;
@@ -111,16 +111,16 @@
   --color-toolkit-checkbox-input-border: #3050ff;
   --color-toolkit-engine-tooltip-border: #ddd;
   --color-toolkit-engine-tooltip-background: #fff;
-  --color-toolkit-loader-border: rgba(0, 0, 0, 0.2);
-  --color-toolkit-loader-borderleft: rgba(255, 255, 255, 0);
+  --color-toolkit-loader-border: rgb(0 0 0 / 20%);
+  --color-toolkit-loader-borderleft: rgb(255 255 255 / 0%);
   --color-doc-code: #003;
   --color-doc-code-background: #ddeaff;
   /// Other misc colors
   --color-bar-chart-primary: #5bc0de;
   --color-bar-chart-secondary: #deb15b;
-  --color-image-resolution-background: rgba(0, 0, 0, 50%);
+  --color-image-resolution-background: rgb(0 0 0 / 50%);
   --color-image-resolution-font: #fff;
-  --color-loading-indicator: rgba(255, 255, 255, 0.2);
+  --color-loading-indicator: rgb(255 255 255 / 20%);
   --color-loading-indicator-gap: #fff;
   --color-line-number: #64708d;
   // Favicons Colors
@@ -200,8 +200,8 @@
   --color-result-detail-background: #1a1a1c;
   --color-result-detail-hr: #555;
   --color-result-detail-link: #8af;
-  --color-result-detail-loader-border: rgba(255, 255, 255, 0.2);
-  --color-result-detail-loader-borderleft: rgba(0, 0, 0, 0);
+  --color-result-detail-loader-border: rgb(255 255 255 / 20%);
+  --color-result-detail-loader-borderleft: rgb(0 0 0 / 0%);
   // Images Colors
   --color-result-image-span-font: #bbb;
   --color-result-image-span-font-selected: #222;
@@ -234,8 +234,8 @@
   --color-toolkit-checkbox-input-border: #58f;
   --color-toolkit-engine-tooltip-border: #333;
   --color-toolkit-engine-tooltip-background: #222;
-  --color-toolkit-loader-border: rgba(255, 255, 255, 0.2);
-  --color-toolkit-loader-borderleft: rgba(0, 0, 0, 0);
+  --color-toolkit-loader-border: rgb(255 255 255 / 20%);
+  --color-toolkit-loader-borderleft: rgb(0 0 0 / 0%);
   --color-doc-code: #ddd;
   --color-doc-code-background: #4d5a6f;
   // Favicons Colors

searx/static/themes/simple/src/less/detail.less

@@ -93,8 +93,7 @@ article.result-images .detail {
 
     p.result-url {
       white-space: nowrap;
-      overflow-x: hidden;
-      overflow-y: hidden;
+      overflow: hidden hidden;
       text-overflow: ellipsis;
     }
 

searx/static/themes/simple/src/less/search.less

@@ -15,8 +15,7 @@
   background: var(--color-header-background);
   border-bottom: 1px solid var(--color-header-border);
   display: grid;
-  column-gap: 1.2rem;
-  row-gap: 1rem;
+  gap: 1rem 1.2rem;
   grid-template-columns: 3rem 1fr;
   grid-template-areas:
     "logo search"
@@ -268,8 +267,7 @@ html.no-js #clear_search.hide_if_nojs {
     #categories {
       width: 100%;
       .ltr-text-align-left();
-      overflow-x: scroll;
-      overflow-y: hidden;
+      overflow: scroll hidden;
       -webkit-overflow-scrolling: touch;
     }
   }
@@ -280,8 +278,7 @@ html.no-js #clear_search.hide_if_nojs {
     width: 100%;
     margin: 0;
     padding: 0.1rem 0 0 0;
-    column-gap: 0;
-    row-gap: 0;
+    gap: 0 0;
     grid-template-areas:
       "logo search"
       "categories categories";

searx/static/themes/simple/src/less/style-center.less

@@ -14,7 +14,7 @@ on the /search URL and when the "center alignment" preference is enabled.
   }
 }
 
-@media screen and (min-width: 62rem) {
+@media screen and (width >= 62rem) {
   .center-alignment-yes #main_results {
     --center-page-width: 60rem;
   }
@@ -91,7 +91,7 @@ on the /search URL and when the "center alignment" preference is enabled.
         align-self: flex-start;
       }
 
-      &:not(.only_template_images):not(.image-detail-open) {
+      &:not(.only_template_images, .image-detail-open) {
         // the gap is set in style.less
         .ltr-margin-left(1.5rem);
         grid-template-columns: calc(var(--center-page-width) - @results-gap - @results-sidebar-width) @results-sidebar-width;

searx/static/themes/simple/src/less/style.less

@@ -244,9 +244,8 @@ article[data-vim-selected].category-social {
     align-items: center;
     font-size: 1rem;
     color: var(--color-result-url-font);
-    flex-wrap: nowrap;
+    flex-flow: row nowrap;
     overflow: hidden;
-    flex-direction: row;
     margin: 0;
     padding: 0;
 
@@ -269,9 +268,7 @@ article[data-vim-selected].category-social {
     .url_o2 {
       overflow: hidden;
       white-space: nowrap;
-      flex-basis: content;
-      flex-grow: 0;
-      flex-shrink: 1;
+      flex: 0 1 content;
       text-align: right;
       padding-bottom: 1px;
 
@@ -546,14 +543,8 @@ article[data-vim-selected].category-social {
   margin-bottom: 0;
   .ltr-margin-left(@results-offset);
   display: grid;
-  grid-template-columns: @results-width @results-sidebar-width;
-  grid-template-rows: min-content min-content 1fr min-content;
+  grid-template: "corrections sidebar" min-content "answers sidebar" min-content "urls sidebar" 1fr "pagination sidebar" min-content / @results-width @results-sidebar-width;
   gap: 0 @results-gap;
-  grid-template-areas:
-    "corrections sidebar"
-    "answers sidebar"
-    "urls sidebar"
-    "pagination sidebar";
 }
 
 #results #sidebar *:first-child {
@@ -676,10 +667,7 @@ summary.title {
   }
 
   .answer-url {
-    margin-left: auto;
-    margin-top: 5px;
-    margin-right: 10px;
-    margin-bottom: 10px;
+    margin: 5px 10px 10px auto;
   }
 }
 
@@ -979,19 +967,12 @@ summary.title {
     margin: 0 auto;
     justify-content: center;
     display: grid;
-    grid-template-columns: @results-width;
-    grid-template-rows: min-content min-content min-content 1fr min-content min-content;
+    grid-template: "corrections" min-content "answers" min-content "sidebar" min-content "urls" 1fr "pagination" min-content min-content / @results-width;
     gap: 0;
-    grid-template-areas:
-      "corrections"
-      "answers"
-      "sidebar"
-      "urls"
-      "pagination";
   }
 }
 
-@media screen and (max-width: calc(@tablet - 0.5px)) {
+@media screen and (width <= calc(@tablet - 0.5px)) {
   #links_on_top {
     span {
       display: none;
@@ -999,7 +980,7 @@ summary.title {
   }
 }
 
-@media screen and (max-width: 52rem) {
+@media screen and (width <= 52rem) {
   body.results_endpoint {
     #links_on_top {
       .link_on_top_about,
@@ -1024,15 +1005,8 @@ summary.title {
 #main_results div#results.only_template_images {
   margin: 1rem @results-tablet-offset 0 @results-tablet-offset;
   display: grid;
-  grid-template-columns: 100%;
-  grid-template-rows: min-content min-content min-content 1fr min-content;
+  grid-template: "corrections" min-content "answers" min-content "sidebar" min-content "urls" 1fr "pagination" min-content / 100%;
   gap: 0;
-  grid-template-areas:
-    "corrections"
-    "answers"
-    "sidebar"
-    "urls"
-    "pagination";
 
   #sidebar {
     display: none;

searx/static/themes/simple/src/less/toolkit.less

@@ -217,8 +217,7 @@ div.selectable_url {
 
 //
 .scrollx {
-  overflow-x: auto;
-  overflow-y: hidden;
+  overflow: auto hidden;
   display: block;
   padding: 0;
   margin: 0;
@@ -505,7 +504,7 @@ input.checkbox-onoff.reversed-checkbox[type="checkbox"] {
   }
 
   // if not checked and possible to checked then display a "light" check mark on hover
-  input.checkbox[type=checkbox]:not(:checked):not([disabled]):not(.checkbox-onoff):hover::after {
+  input.checkbox[type=checkbox]:not(:checked, [disabled], .checkbox-onoff):hover::after {
     opacity: 0.5;
   }
 }
@@ -593,8 +592,7 @@ td:hover .engine-tooltip,
   width: -moz-available;
   width: -webkit-fill-available;
   width: fill;
-  flex-direction: row;
-  flex-wrap: nowrap;
+  flex-flow: row nowrap;
   align-items: center;
   display: inline-flex;
 }

searx/static/themes/simple/src/less/toolkit_loader.less

@@ -14,33 +14,16 @@
   border-right: 1.1em solid var(--color-loading-indicator);
   border-bottom: 1.1em solid var(--color-loading-indicator);
   border-left: 1.1em solid var(--color-loading-indicator-gap);
-  -webkit-transform: translateZ(0);
-  -ms-transform: translateZ(0);
   transform: translateZ(0);
-  -webkit-animation: load8 1.1s infinite linear;
   animation: load8 1.1s infinite linear;
 }
 
-@-webkit-keyframes load8 {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg);
-  }
-
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg);
-  }
-}
-
 @keyframes load8 {
   0% {
-    -webkit-transform: rotate(0deg);
     transform: rotate(0deg);
   }
 
   100% {
-    -webkit-transform: rotate(360deg);
     transform: rotate(360deg);
   }
 }

tests/unit/engines/test_json_engine.py

@@ -0,0 +1,254 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# pylint: disable=missing-module-docstring
+
+from collections import defaultdict
+import mock
+
+from searx.engines import json_engine
+from searx import logger
+
+from tests import SearxTestCase
+
+logger = logger.getChild('engines')
+
+
+class TestJsonEngine(SearxTestCase):  # pylint: disable=missing-class-docstring
+    json = """
+    [
+        {
+            "title": "title0",
+            "content": "content0",
+            "url": "https://example.com/url0",
+            "images": [
+                {
+                    "thumb": "https://example.com/thumb00"
+                },
+                {
+                    "thumb": "https://example.com/thumb01"
+                }
+            ]
+        },
+        {
+            "title": "<h1>title1</h1>",
+            "content": "<h2>content1</h2>",
+            "url": "https://example.com/url1",
+            "images": [
+                {
+                    "thumb": "https://example.com/thumb10"
+                },
+                {
+                    "thumb": "https://example.com/thumb11"
+                }
+            ]
+        },
+        {
+            "title": "title2",
+            "content": "content2",
+            "url": 2,
+            "images": [
+                {
+                    "thumb": "thumb20"
+                },
+                {
+                    "thumb": 21
+                }
+            ]
+        }
+    ]
+    """
+
+    json_result_query = """
+    {
+        "data": {
+            "results": [
+                {
+                    "title": "title0",
+                    "content": "content0",
+                    "url": "https://example.com/url0",
+                    "images": [
+                        {
+                            "thumb": "https://example.com/thumb00"
+                        },
+                        {
+                            "thumb": "https://example.com/thumb01"
+                        }
+                    ]
+                },
+                {
+                    "title": "<h1>title1</h1>",
+                    "content": "<h2>content1</h2>",
+                    "url": "https://example.com/url1",
+                    "images": [
+                        {
+                            "thumb": "https://example.com/thumb10"
+                        },
+                        {
+                            "thumb": "https://example.com/thumb11"
+                        }
+                    ]
+                },
+                {
+                    "title": "title2",
+                    "content": "content2",
+                    "url": 2,
+                    "images": [
+                        {
+                            "thumb": "thumb20"
+                        },
+                        {
+                            "thumb": 21
+                        }
+                    ]
+                }
+            ],
+            "suggestions": [
+                "suggestion0",
+                "suggestion1"
+            ]
+        }
+    }
+    """
+
+    def setUp(self):
+        json_engine.logger = logger.getChild('test_json_engine')
+
+    def test_request(self):
+        json_engine.search_url = 'https://example.com/{query}'
+        json_engine.categories = []
+        json_engine.paging = False
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['language'] = 'all'
+        dicto['pageno'] = 1
+        params = json_engine.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertEqual('https://example.com/test_query', params['url'])
+
+        json_engine.search_url = 'https://example.com/q={query}&p={pageno}'
+        json_engine.paging = True
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['language'] = 'all'
+        dicto['pageno'] = 1
+        params = json_engine.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertEqual('https://example.com/q=test_query&p=1', params['url'])
+
+        json_engine.search_url = 'https://example.com/'
+        json_engine.paging = True
+        json_engine.request_body = '{{"page": {pageno}, "query": "{query}"}}'
+        query = 'test_query'
+        dicto = defaultdict(dict)
+        dicto['language'] = 'all'
+        dicto['pageno'] = 1
+        params = json_engine.request(query, dicto)
+        self.assertIn('data', params)
+        self.assertEqual('{"page": 1, "query": "test_query"}', params['data'])
+
+    def test_response(self):
+        # without results_query
+        json_engine.results_query = ''
+        json_engine.url_query = 'url'
+        json_engine.url_prefix = ''
+        json_engine.title_query = 'title'
+        json_engine.content_query = 'content'
+        json_engine.thumbnail_query = 'images/thumb'
+        json_engine.thumbnail_prefix = ''
+        json_engine.title_html_to_text = False
+        json_engine.content_html_to_text = False
+        json_engine.categories = []
+
+        self.assertRaises(AttributeError, json_engine.response, None)
+        self.assertRaises(AttributeError, json_engine.response, [])
+        self.assertRaises(AttributeError, json_engine.response, '')
+        self.assertRaises(AttributeError, json_engine.response, '[]')
+
+        response = mock.Mock(text='{}', status_code=200)
+        self.assertEqual(json_engine.response(response), [])
+
+        response = mock.Mock(text=self.json, status_code=200)
+        results = json_engine.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 3)
+        self.assertEqual(results[0]['title'], 'title0')
+        self.assertEqual(results[0]['url'], 'https://example.com/url0')
+        self.assertEqual(results[0]['content'], 'content0')
+        self.assertEqual(results[0]['thumbnail'], 'https://example.com/thumb00')
+        self.assertEqual(results[1]['title'], '<h1>title1</h1>')
+        self.assertEqual(results[1]['url'], 'https://example.com/url1')
+        self.assertEqual(results[1]['content'], '<h2>content1</h2>')
+        self.assertEqual(results[1]['thumbnail'], 'https://example.com/thumb10')
+
+        # with prefix and suggestions without results_query
+        json_engine.url_prefix = 'https://example.com/url'
+        json_engine.thumbnail_query = 'images/1/thumb'
+        json_engine.thumbnail_prefix = 'https://example.com/thumb'
+
+        results = json_engine.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 3)
+        self.assertEqual(results[2]['title'], 'title2')
+        self.assertEqual(results[2]['url'], 'https://example.com/url2')
+        self.assertEqual(results[2]['content'], 'content2')
+        self.assertEqual(results[2]['thumbnail'], 'https://example.com/thumb21')
+        self.assertFalse(results[0].get('is_onion', False))
+
+        # results are onion urls without results_query
+        json_engine.categories = ['onions']
+        results = json_engine.response(response)
+        self.assertTrue(results[0]['is_onion'])
+
+    def test_response_results_json(self):
+        # with results_query
+        json_engine.results_query = 'data/results'
+        json_engine.url_query = 'url'
+        json_engine.url_prefix = ''
+        json_engine.title_query = 'title'
+        json_engine.content_query = 'content'
+        json_engine.thumbnail_query = 'images/1/thumb'
+        json_engine.thumbnail_prefix = ''
+        json_engine.title_html_to_text = True
+        json_engine.content_html_to_text = True
+        json_engine.categories = []
+
+        self.assertRaises(AttributeError, json_engine.response, None)
+        self.assertRaises(AttributeError, json_engine.response, [])
+        self.assertRaises(AttributeError, json_engine.response, '')
+        self.assertRaises(AttributeError, json_engine.response, '[]')
+
+        response = mock.Mock(text='{}', status_code=200)
+        self.assertEqual(json_engine.response(response), [])
+
+        response = mock.Mock(text=self.json_result_query, status_code=200)
+        results = json_engine.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 3)
+        self.assertEqual(results[0]['title'], 'title0')
+        self.assertEqual(results[0]['url'], 'https://example.com/url0')
+        self.assertEqual(results[0]['content'], 'content0')
+        self.assertEqual(results[0]['thumbnail'], 'https://example.com/thumb01')
+        self.assertEqual(results[1]['title'], 'title1')
+        self.assertEqual(results[1]['url'], 'https://example.com/url1')
+        self.assertEqual(results[1]['content'], 'content1')
+        self.assertEqual(results[1]['thumbnail'], 'https://example.com/thumb11')
+
+        # with prefix and suggestions with results_query
+        json_engine.url_prefix = 'https://example.com/url'
+        json_engine.thumbnail_query = 'images/1/thumb'
+        json_engine.thumbnail_prefix = 'https://example.com/thumb'
+        json_engine.suggestion_query = 'data/suggestions'
+
+        results = json_engine.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 4)
+        self.assertEqual(results[2]['title'], 'title2')
+        self.assertEqual(results[2]['url'], 'https://example.com/url2')
+        self.assertEqual(results[2]['content'], 'content2')
+        self.assertEqual(results[2]['thumbnail'], 'https://example.com/thumb21')
+        self.assertEqual(results[3]['suggestion'], ['suggestion0', 'suggestion1'])
+        self.assertFalse(results[0].get('is_onion', False))
+
+        # results are onion urls with results_query
+        json_engine.categories = ['onions']
+        results = json_engine.response(response)
+        self.assertTrue(results[0]['is_onion'])

utils/lib_sxng_node.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+declare _Blue
+declare _creset
+
 export NODE_MINIMUM_VERSION="16.13.0"
 
 node.help(){
@@ -22,7 +25,7 @@ nodejs.ensure() {
 node.env() {
     nodejs.ensure
     (   set -e
-        build_msg INSTALL "./searx/static/themes/simple/package.json"
+        build_msg INSTALL "[npm] ./searx/static/themes/simple/package.json"
         npm --prefix searx/static/themes/simple install
     )
     dump_return $?
@@ -30,7 +33,7 @@ node.env() {
 
 node.env.dev() {
     nodejs.ensure
-    build_msg INSTALL "./package.json: developer and CI tools"
+    build_msg INSTALL "[npm] ./package.json: developer and CI tools"
     npm install
 }
 
@@ -41,11 +44,19 @@ node.clean() {
     fi
     build_msg CLEAN "themes -- locally installed npm dependencies"
     (   set -e
-        npm --prefix searx/static/themes/simple run clean
+        npm --prefix searx/static/themes/simple run clean \
+	    | prefix_stdout "${_Blue}CLEAN    ${_creset} "
+	if [ "${PIPESTATUS[0]}" -ne "0" ]; then
+            return 1
+	fi
     )
     build_msg CLEAN "locally installed developer and CI tools"
     (   set -e
-        npm --prefix . run clean
+        npm --prefix . run clean \
+	    | prefix_stdout "${_Blue}CLEAN    ${_creset} "
+	if [ "${PIPESTATUS[0]}" -ne "0" ]; then
+            return 1
+	fi
     )
     dump_return $?
 }

utils/lib_sxng_themes.sh

@@ -9,16 +9,14 @@ themes.help(){
 themes.:
   all       : build all themes
   live      : to get live builds of CSS & JS use 'LIVE_THEME=simple make run'
-  simple.:
-    build   : build simple theme
+  simple.:    build simple theme
     test    : test simple theme
+    pygments: build pygment's LESS files for simple theme
 EOF
 }
 
 themes.all() {
     (   set -e
-        pygments.less
-        node.env
         themes.simple
     )
     dump_return $?
@@ -50,15 +48,29 @@ themes.live() {
 
 themes.simple() {
     (   set -e
-        build_msg GRUNT "theme: simple"
-        npm --prefix searx/static/themes/simple run build
+	node.env
+	themes.simple.pygments
     )
+    build_msg GRUNT "theme: simple"
+    npm --prefix searx/static/themes/simple run build
     dump_return $?
 }
 
+themes.simple.pygments() {
+    build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
+    pyenv.cmd python searxng_extra/update/update_pygments.py \
+	| prefix_stdout "${_Blue}PYGMENTS ${_creset} "
+    if [ "${PIPESTATUS[0]}" -ne "0" ]; then
+        build_msg PYGMENTS "building LESS files for pygments failed"
+        return 1
+    fi
+    return 0
+}
+
+
 themes.simple.test() {
     build_msg TEST "theme: simple"
-    nodejs.ensure
+    node.env
     npm --prefix searx/static/themes/simple install
     npm --prefix searx/static/themes/simple run test
     dump_return $?