[fix] searx.data: fetch-traits - z-library (httpx.ConnectError) (#4835)

There is currently no known z-library, and all known URLs are dead [1]. To avoid
interrupting automated updates, a connection error to a z-library is treated as
a *known error*, and the old properties of the z-library are retained.

[1] https://github.com/searxng/searxng/issues/3610

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

client/simple/package-lock.json

@@ -7,9 +7,6 @@
     "": {
       "name": "simple",
       "version": "1.0.0",
-      "dependencies": {
-        "autocomplete-js": "^2.7.1"
-      },
       "devDependencies": {
         "@eslint/js": "^9.26.0",
         "copy-webpack-plugin": "^13.0.0",
@@ -2526,12 +2523,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/autocomplete-js": {
-      "version": "2.7.1",
-      "resolved": "https://registry.npmjs.org/autocomplete-js/-/autocomplete-js-2.7.1.tgz",
-      "integrity": "sha512-F+1Sj13zbfpbkprKjtyYiaHgBo6YODzNzWgJjgo/p312AeiZVEn3W+NaRRzVZYqMI0Bm7/W36uanp995eNDy+A==",
-      "license": "MIT"
-    },
     "node_modules/balanced-match": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",

client/simple/package.json

@@ -34,8 +34,5 @@
     "vite-plugin-stylelint": "^6.0.0",
     "webpack": "^5.99.8",
     "webpack-cli": "^6.0.1"
-  },
-  "dependencies": {
-    "autocomplete-js": "^2.7.1"
   }
 }

client/simple/src/js/main/search.js

@@ -1,8 +1,6 @@
 /* SPDX-License-Identifier: AGPL-3.0-or-later */
 /* exported AutoComplete */
 
-import AutoComplete from  "../../../node_modules/autocomplete-js/dist/autocomplete.js";
-
 (function (w, d, searxng) {
   'use strict';
 
@@ -38,8 +36,62 @@ import AutoComplete from  "../../../node_modules/autocomplete-js/dist/autocomple
     qinput.addEventListener('input', updateClearButton, false);
   }
 
+  const fetchResults = async (query) => {
+    let request;
+    if (searxng.settings.method === 'GET') {
+      const reqParams = new URLSearchParams();
+      reqParams.append("q", query);
+      request = fetch("./autocompleter?" + reqParams.toString());
+    } else {
+      const formData = new FormData();
+      formData.append("q", query);
+      request = fetch("./autocompleter", {
+        method: 'POST',
+        body: formData,
+      });
+    }
+
+    request.then(async function (response) {
+      const results = await response.json();
+
+      if (!results) return;
+
+      const autocomplete = d.querySelector(".autocomplete");
+      const autocompleteList = d.querySelector(".autocomplete ul");
+      autocomplete.classList.add("open");
+      autocompleteList.innerHTML = "";
+
+      // show an error message that no result was found
+      if (!results[1] || results[1].length == 0) {
+        const noItemFoundMessage = document.createElement("li");
+        noItemFoundMessage.classList.add('no-item-found');
+        noItemFoundMessage.innerHTML = searxng.settings.translations.no_item_found;
+        autocompleteList.appendChild(noItemFoundMessage);
+        return;
+      }
+
+      for (let result of results[1]) {
+        const li = document.createElement("li");
+        li.innerText = result;
+
+        searxng.on(li, 'mousedown', () => {
+          qinput.value = result;
+          const form = d.querySelector("#search");
+          form.submit();
+          autocomplete.classList.remove('open');
+        });
+        autocompleteList.appendChild(li);
+      }
+    });
+  };
+
   searxng.ready(function () {
+    // focus search input on large screens
+    if (!isMobile) document.getElementById("q").focus();
+
     qinput = d.getElementById(qinput_id);
+    const autocomplete = d.querySelector(".autocomplete");
+    const autocompleteList = d.querySelector(".autocomplete ul");
 
     if (qinput !== null) {
       // clear button
@@ -47,109 +99,45 @@ import AutoComplete from  "../../../node_modules/autocomplete-js/dist/autocomple
 
       // autocompleter
       if (searxng.settings.autocomplete) {
-        searxng.autocomplete = AutoComplete.call(w, {
+        searxng.on(qinput, 'input', () => {
-          Url: "./autocompleter",
+          const query = qinput.value;
-          EmptyMessage: searxng.settings.translations.no_item_found,
+          if (query.length < searxng.settings.autocomplete_min) return;
-          HttpMethod: searxng.settings.method,
+
-          HttpHeaders: {
+          setTimeout(() => {
-            "Content-type": "application/x-www-form-urlencoded",
+            if (query == qinput.value) fetchResults(query);
-            "X-Requested-With": "XMLHttpRequest"
+          }, 300);
-          },
+        });
-          MinChars: searxng.settings.autocomplete_min,
+
-          Delay: 300,
+        searxng.on(qinput, 'keyup', (e) => {
-          _Position: function () {},
+          let currentIndex = -1;
-          _Open: function () {
+          const listItems = autocompleteList.children;
-            var params = this;
+          for (let i = 0; i < listItems.length; i++) {
-            Array.prototype.forEach.call(this.DOMResults.getElementsByTagName("li"), function (li) {
+            if (listItems[i].classList.contains('active')) {
-              if (li.getAttribute("class") != "locked") {
+              currentIndex = i;
-                li.onmousedown = function () {
+              break;
-                  params._Select(li);
-                };
-              }
-            });
-          },
-          _Select: function (item) {
-            AutoComplete.defaults._Select.call(this, item);
-            var form = item.closest('form');
-            if (form) {
-              form.submit();
             }
-          },
-          _MinChars: function () {
-            if (this.Input.value.indexOf('!') > -1) {
-              return 0;
-            } else {
-              return AutoComplete.defaults._MinChars.call(this);
-            }
-          },
-          KeyboardMappings: Object.assign({}, AutoComplete.defaults.KeyboardMappings, {
-            "KeyUpAndDown_up": Object.assign({}, AutoComplete.defaults.KeyboardMappings.KeyUpAndDown_up, {
-              Callback: function (event) {
-                AutoComplete.defaults.KeyboardMappings.KeyUpAndDown_up.Callback.call(this, event);
-                var liActive = this.DOMResults.querySelector("li.active");
-                if (liActive) {
-                  AutoComplete.defaults._Select.call(this, liActive);
-                }
-              },
-            }),
-            "Tab": Object.assign({}, AutoComplete.defaults.KeyboardMappings.Enter, {
-              Conditions: [{
-                Is: 9,
-                Not: false
-              }],
-              Callback: function (event) {
-                if (this.DOMResults.getAttribute("class").indexOf("open") != -1) {
-                  var liActive = this.DOMResults.querySelector("li.active");
-                  if (liActive !== null) {
-                    AutoComplete.defaults._Select.call(this, liActive);
-                    event.preventDefault();
-                  }
-                }
-              },
-            })
-          }),
-        }, "#" + qinput_id);
-      }
-
-      /*
-        Monkey patch autocomplete.js to fix a bug
-        With the POST method, the values are not URL encoded: query like "1 + 1" are sent as "1  1" since space are URL encoded as plus.
-        See HTML specifications:
-        * HTML5: https://url.spec.whatwg.org/#concept-urlencoded-serializer
-        * HTML4: https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1
-
-        autocomplete.js does not URL encode the name and values:
-        https://github.com/autocompletejs/autocomplete.js/blob/87069524f3b95e68f1b54d8976868e0eac1b2c83/src/autocomplete.ts#L665
-
-        The monkey patch overrides the compiled version of the ajax function.
-        See https://github.com/autocompletejs/autocomplete.js/blob/87069524f3b95e68f1b54d8976868e0eac1b2c83/dist/autocomplete.js#L143-L158
-        The patch changes only the line 156 from
-          params.Request.send(params._QueryArg() + "=" + params._Pre());
-        to
-          params.Request.send(encodeURIComponent(params._QueryArg()) + "=" + encodeURIComponent(params._Pre()));
-
-        Related to:
-        * https://github.com/autocompletejs/autocomplete.js/issues/78
-        * https://github.com/searxng/searxng/issues/1695
-       */
-      AutoComplete.prototype.ajax = function (params, request, timeout) {
-        if (timeout === void 0) { timeout = true; }
-        if (params.$AjaxTimer) {
-          window.clearTimeout(params.$AjaxTimer);
-        }
-        if (timeout === true) {
-          params.$AjaxTimer = window.setTimeout(AutoComplete.prototype.ajax.bind(null, params, request, false), params.Delay);
-        } else {
-          if (params.Request) {
-            params.Request.abort();
           }
-          params.Request = request;
-          params.Request.send(encodeURIComponent(params._QueryArg()) + "=" + encodeURIComponent(params._Pre()));
-        }
-      };
 
-      if (!isMobile && document.querySelector('.index_endpoint')) {
+          let newCurrentIndex = -1;
-        qinput.focus();
+          if (e.key === "ArrowUp") {
+            if (currentIndex >= 0) listItems[currentIndex].classList.remove('active');
+            // we need to add listItems.length to the index calculation here because the JavaScript modulos
+            // operator doesn't work with negative numbers
+            newCurrentIndex = (currentIndex - 1 + listItems.length) % listItems.length;
+          } else if (e.key === "ArrowDown") {
+            if (currentIndex >= 0) listItems[currentIndex].classList.remove('active');
+            newCurrentIndex = (currentIndex + 1) % listItems.length;
+          } else if (e.key === "Tab" || e.key === "Enter") {
+            autocomplete.classList.remove('open');
+          }
+
+          if (newCurrentIndex != -1) {
+            const selectedItem = listItems[newCurrentIndex];
+            selectedItem.classList.add('active');
+
+            if (!selectedItem.classList.contains('no-item-found')) qinput.value = selectedItem.innerText;
+          }
+        });
       }
     }
 
@@ -184,7 +172,7 @@ import AutoComplete from  "../../../node_modules/autocomplete-js/dist/autocomple
           categoryButton.classList.remove("selected");
         }
         button.classList.add("selected");
-      })
+      });
     }
 
     // override form submit action to update the actually selected categories

client/simple/src/less/autocomplete.less

@@ -3,6 +3,7 @@
 .autocomplete {
   position: absolute;
   width: @search-width;
+  max-width: calc(100% - 2 * @search-padding-horizontal);
   max-height: 0;
   overflow-y: hidden;
   .ltr-text-align-left();
@@ -65,8 +66,6 @@
 
 @media screen and (max-width: @phone) {
   .autocomplete {
-    width: 100%;
-
     > ul > li {
       padding: 1rem;
     }

client/simple/src/less/definitions.less

@@ -287,8 +287,9 @@
 @results-image-row-height: 12rem;
 @results-image-row-height-phone: 10rem;
 @search-width: 44rem;
-// heigh of #search, see detail.less
+// height of #search, see detail.less
 @search-height: 13rem;
+@search-padding-horizontal: 0.5rem;
 
 /// Device Size
 /// @desktop > @tablet

client/simple/src/less/search.less

@@ -131,7 +131,7 @@ button.category_button {
 }
 
 #search_view {
-  padding: 0.5rem 0.3rem 0 0.5rem;
+  padding: 0.5rem @search-padding-horizontal 0 @search-padding-horizontal;
   grid-area: search;
 
   body.results_endpoint & {
@@ -141,7 +141,8 @@ button.category_button {
 
 .search_box {
   border-radius: 0.8rem;
-  width: @search-width;
+  width: 100%;
+  max-width: @search-width;
   display: inline-flex;
   flex-direction: row;
   white-space: nowrap;
@@ -291,8 +292,7 @@ html.no-js #clear_search.hide_if_nojs {
   }
 
   .search_box {
-    width: 98%;
+    width: 100%;
-    display: flex;
   }
 
   #q {

searx/data/__init__.py

@@ -5,28 +5,68 @@
 
 """
 
-__all__ = [
+
-    'ENGINE_TRAITS',
+__all__ = ["ahmia_blacklist_loader"]
-    'CURRENCIES',
-    'USER_AGENTS',
-    'EXTERNAL_URLS',
-    'WIKIDATA_UNITS',
-    'EXTERNAL_BANGS',
-    'OSM_KEYS_TAGS',
-    'ENGINE_DESCRIPTIONS',
-    'LOCALES',
-    'ahmia_blacklist_loader',
-]
 
 import json
 from pathlib import Path
+import typing
+
+from searx import logger
+
+log = logger.getChild("data")
 
 data_dir = Path(__file__).parent
 
+CURRENCIES: dict[str, typing.Any]
+USER_AGENTS: dict[str, typing.Any]
+EXTERNAL_URLS: dict[str, typing.Any]
+WIKIDATA_UNITS: dict[str, typing.Any]
+EXTERNAL_BANGS: dict[str, typing.Any]
+OSM_KEYS_TAGS: dict[str, typing.Any]
+ENGINE_DESCRIPTIONS: dict[str, typing.Any]
+ENGINE_TRAITS: dict[str, typing.Any]
+LOCALES: dict[str, typing.Any]
 
-def _load(filename):
+lazy_globals = {
-    with open(data_dir / filename, encoding='utf-8') as f:
+    "CURRENCIES": None,
-        return json.load(f)
+    "USER_AGENTS": None,
+    "EXTERNAL_URLS": None,
+    "WIKIDATA_UNITS": None,
+    "EXTERNAL_BANGS": None,
+    "OSM_KEYS_TAGS": None,
+    "ENGINE_DESCRIPTIONS": None,
+    "ENGINE_TRAITS": None,
+    "LOCALES": None,
+}
+
+data_json_files = {
+    "CURRENCIES": "currencies.json",
+    "USER_AGENTS": "useragents.json",
+    "EXTERNAL_URLS": "external_urls.json",
+    "WIKIDATA_UNITS": "wikidata_units.json",
+    "EXTERNAL_BANGS": "external_bangs.json",
+    "OSM_KEYS_TAGS": "osm_keys_tags.json",
+    "ENGINE_DESCRIPTIONS": "engine_descriptions.json",
+    "ENGINE_TRAITS": "engine_traits.json",
+    "LOCALES": "locales.json",
+}
+
+
+def __getattr__(name):
+    # lazy init of the global objects
+    if name not in lazy_globals:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+    data = lazy_globals[name]
+    if data is not None:
+        return data
+
+    log.debug("init searx.data.%s", name)
+    with open(data_dir / data_json_files[name], encoding='utf-8') as f:
+        lazy_globals[name] = json.load(f)
+
+    return lazy_globals[name]
 
 
 def ahmia_blacklist_loader():
@@ -40,14 +80,3 @@ def ahmia_blacklist_loader():
     """
     with open(data_dir / 'ahmia_blacklist.txt', encoding='utf-8') as f:
         return f.read().split()
-
-
-CURRENCIES = _load('currencies.json')
-USER_AGENTS = _load('useragents.json')
-EXTERNAL_URLS = _load('external_urls.json')
-WIKIDATA_UNITS = _load('wikidata_units.json')
-EXTERNAL_BANGS = _load('external_bangs.json')
-OSM_KEYS_TAGS = _load('osm_keys_tags.json')
-ENGINE_DESCRIPTIONS = _load('engine_descriptions.json')
-ENGINE_TRAITS = _load('engine_traits.json')
-LOCALES = _load('locales.json')

searx/engines/zlibrary.py

@@ -180,6 +180,8 @@ def fetch_traits(engine_traits: EngineTraits) -> None:
     # pylint: disable=import-outside-toplevel, too-many-branches
 
     import babel
+    import httpx
+
     from searx.network import get  # see https://github.com/searxng/searxng/issues/762
     from searx.locales import language_tag
 
@@ -191,7 +193,7 @@ def fetch_traits(engine_traits: EngineTraits) -> None:
 
     try:
         resp = get(base_url, verify=False)
-    except SearxException as exc:
+    except (SearxException, httpx.ConnectError) as exc:
         print(f"ERROR: zlibrary domain '{base_url}' is seized?")
         print(f"  --> {exc}")
         _use_old_values()

searx/templates/simple/search.html

@@ -9,6 +9,7 @@
         <input id="q" name="q" type="text" placeholder="{{ _('Search for...') }}" tabindex="1" autocomplete="off" autocapitalize="none" spellcheck="false" autocorrect="off" dir="auto" value="{{ q or '' }}">
         <button id="clear_search" type="reset" aria-label="{{ _('clear') }}" class="hide_if_nojs"><span>{{ icon_big('close') }}</span><span class="show_if_nojs">{{ _('clear') }}</span></button>
         <button id="send_search" type="submit" {%- if search_on_category_select -%}name="category_{{ selected_categories[0]|replace(' ', '_') }}"{%- endif -%} aria-label="{{ _('search') }}"><span class="hide_if_nojs">{{ icon_big('search') }}</span><span class="show_if_nojs">{{ _('search') }}</span></button>
+        <div class="autocomplete hide_if_nojs"><ul></ul></div>
       </div>
     </div>
     {% set display_tooltip = true %}

searx/templates/simple/simple_search.html

@@ -5,6 +5,7 @@
         <input id="q" name="q" type="text" placeholder="{{ _('Search for...') }}" autocomplete="off" autocapitalize="none" spellcheck="false" autocorrect="off" dir="auto" value="{{ q or '' }}">
         <button id="clear_search" type="reset" aria-label="{{ _('clear') }}"><span class="hide_if_nojs">{{ icon_big('close') }}</span><span class="show_if_nojs">{{ _('clear') }}</span></button>
         <button id="send_search" type="submit" aria-label="{{ _('search') }}"><span class="hide_if_nojs">{{ icon_big('search') }}</span><span class="show_if_nojs">{{ _('search') }}</span></button>
+        <div class="autocomplete hide_if_nojs"><ul></ul></div>
       </div>
     </div>
   </div>