[chore] docs: various grammatical fixes

the engine is currently broken because
- it requires a normal user agent (spoofed)
- it has some additional fingerprinting if using http2

searx/cache.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 __all__ = ["ExpireCacheCfg", "ExpireCacheStats", "ExpireCache", "ExpireCacheSQLite"]
 
 import abc
+from collections.abc import Iterator
 import dataclasses
 import datetime
 import hashlib
@@ -396,6 +397,20 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
 
         return self.deserialize(row[0])
 
+    def pairs(self, ctx: str) -> Iterator[tuple[str, typing.Any]]:
+        """Iterate over key/value pairs from table given by argument ``ctx``.
+        If ``ctx`` argument is ``None`` (the default), a table name is
+        generated from the :py:obj:`ExpireCacheCfg.name`."""
+        table = ctx
+        self.maintenance()
+
+        if not table:
+            table = self.normalize_name(self.cfg.name)
+
+        if table in self.table_names:
+            for row in self.DB.execute(f"SELECT key, value FROM {table}"):
+                yield row[0], self.deserialize(row[1])
+
     def state(self) -> ExpireCacheStats:
         cached_items = {}
         for table in self.table_names:

searx/data/__init__.py

@@ -13,6 +13,7 @@ import typing
 
 from .core import log, data_dir
 from .currencies import CurrenciesDB
+from .tracker_patterns import TrackerPatternsDB
 
 CURRENCIES: CurrenciesDB
 USER_AGENTS: dict[str, typing.Any]
@@ -23,6 +24,7 @@ OSM_KEYS_TAGS: dict[str, typing.Any]
 ENGINE_DESCRIPTIONS: dict[str, typing.Any]
 ENGINE_TRAITS: dict[str, typing.Any]
 LOCALES: dict[str, typing.Any]
+TRACKER_PATTERNS: TrackerPatternsDB
 
 lazy_globals = {
     "CURRENCIES": CurrenciesDB(),
@@ -34,6 +36,7 @@ lazy_globals = {
     "ENGINE_DESCRIPTIONS": None,
     "ENGINE_TRAITS": None,
     "LOCALES": None,
+    "TRACKER_PATTERNS": TrackerPatternsDB(),
 }
 
 data_json_files = {

searx/data/tracker_patterns.py

@@ -0,0 +1,142 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Simple implementation to store TrackerPatterns data in a SQL database."""
+
+from __future__ import annotations
+import typing
+
+__all__ = ["TrackerPatternsDB"]
+
+import re
+import pathlib
+from collections.abc import Iterator
+from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
+
+import httpx
+
+from searx.data.core import get_cache, log
+
+RuleType = tuple[str, list[str], list[str]]
+
+
+class TrackerPatternsDB:
+    # pylint: disable=missing-class-docstring
+
+    ctx_name = "data_tracker_patterns"
+    json_file = pathlib.Path(__file__).parent / "tracker_patterns.json"
+
+    CLEAR_LIST_URL = [
+        # ClearURL rule lists, the first one that responds HTTP 200 is used
+        "https://rules1.clearurls.xyz/data.minify.json",
+        "https://rules2.clearurls.xyz/data.minify.json",
+        "https://raw.githubusercontent.com/ClearURLs/Rules/refs/heads/master/data.min.json",
+    ]
+
+    class Fields:
+        # pylint: disable=too-few-public-methods, invalid-name
+        url_regexp: typing.Final = 0  # URL (regular expression) match condition of the link
+        url_ignore: typing.Final = 1  # URL (regular expression) to ignore
+        del_args: typing.Final = 2  # list of URL arguments (regular expression) to delete
+
+    def __init__(self):
+        self.cache = get_cache()
+
+    def init(self):
+        if self.cache.properties("tracker_patterns loaded") != "OK":
+            self.load()
+            self.cache.properties.set("tracker_patterns loaded", "OK")
+        # F I X M E:
+        #     do we need a maintenance .. rember: database is stored
+        #     in /tmp and will be rebuild during the reboot anyway
+
+    def load(self):
+        log.debug("init searx.data.TRACKER_PATTERNS")
+        for rule in self.iter_clear_list():
+            self.add(rule)
+
+    def add(self, rule: RuleType):
+        self.cache.set(
+            key=rule[self.Fields.url_regexp],
+            value=(
+                rule[self.Fields.url_ignore],
+                rule[self.Fields.del_args],
+            ),
+            ctx=self.ctx_name,
+            expire=None,
+        )
+
+    def rules(self) -> Iterator[RuleType]:
+        self.init()
+        for key, value in self.cache.pairs(ctx=self.ctx_name):
+            yield key, value[0], value[1]
+
+    def iter_clear_list(self) -> Iterator[RuleType]:
+        resp = None
+        for url in self.CLEAR_LIST_URL:
+            resp = httpx.get(url, timeout=3)
+            if resp.status_code == 200:
+                break
+            log.warning(f"TRACKER_PATTERNS: ClearURL ignore HTTP {resp.status_code} {url}")
+
+        if resp is None:
+            log.error("TRACKER_PATTERNS: failed fetching ClearURL rule lists")
+            return
+
+        for rule in resp.json()["providers"].values():
+            yield (
+                rule["urlPattern"].replace("\\\\", "\\"),  # fix javascript regex syntax
+                [exc.replace("\\\\", "\\") for exc in rule.get("exceptions", [])],
+                rule.get("rules", []),
+            )
+
+    def clean_url(self, url: str) -> bool | str:
+        """The URL arguments are normalized and cleaned of tracker parameters.
+
+        Returns bool ``True`` to use URL unchanged (``False`` to ignore URL).
+        If URL should be modified, the returned string is the new URL to use.
+        """
+
+        new_url = url
+        parsed_new_url = urlparse(url=new_url)
+
+        for rule in self.rules():
+
+            if not re.match(rule[self.Fields.url_regexp], new_url):
+                # no match / ignore pattern
+                continue
+
+            do_ignore = False
+            for pattern in rule[self.Fields.url_ignore]:
+                if re.match(pattern, new_url):
+                    do_ignore = True
+                    break
+
+            if do_ignore:
+                # pattern is in the list of exceptions / ignore pattern
+                # HINT:
+                #    we can't break the outer pattern loop since we have
+                #    overlapping urlPattern like ".*"
+                continue
+
+            # remove tracker arguments from the url-query part
+            query_args: list[tuple[str, str]] = list(parse_qsl(parsed_new_url.query))
+
+            for name, val in query_args.copy():
+                # remove URL arguments
+                for pattern in rule[self.Fields.del_args]:
+                    if re.match(pattern, name):
+                        log.debug("TRACKER_PATTERNS: %s remove tracker arg: %s='%s'", parsed_new_url.netloc, name, val)
+                        query_args.remove((name, val))
+
+            parsed_new_url = parsed_new_url._replace(query=urlencode(query_args))
+            new_url = urlunparse(parsed_new_url)
+
+        if new_url != url:
+            return new_url
+
+        return True
+
+
+if __name__ == "__main__":
+    db = TrackerPatternsDB()
+    for r in db.rules():
+        print(r)

searx/engines/uxwing.py

@@ -4,7 +4,7 @@
 from urllib.parse import quote_plus
 from lxml import html
 
-from searx.utils import eval_xpath, eval_xpath_list, extract_text
+from searx.utils import eval_xpath, eval_xpath_list, extract_text, gen_useragent
 
 about = {
     "website": 'https://uxwing.com',
@@ -17,10 +17,12 @@ about = {
 categories = ['images', 'icons']
 
 base_url = "https://uxwing.com"
+enable_http2 = False
 
 
 def request(query, params):
     params['url'] = f"{base_url}/?s={quote_plus(query)}"
+    params['headers'] = {'User-Agent': gen_useragent()}
     return params
 
 

searx/infopage/en/about.md

@@ -4,10 +4,10 @@ SearXNG is a [metasearch engine], aggregating the results of other
 {{link('search engines', 'preferences')}} while not storing information about
 its users.
 
-The SearXNG project is driven by an open community, come join us on Matrix if
+The SearXNG project is driven by an open community. Come join us on Matrix if
 you have questions or just want to chat about SearXNG at [#searxng:matrix.org]
 
-Make SearXNG better.
+Make SearXNG better:
 
 - You can improve SearXNG translations at [Weblate], or...
 - Track development, send contributions, and report issues at [SearXNG sources].
@@ -19,8 +19,8 @@ Make SearXNG better.
 - SearXNG may not offer you as personalized results as Google, but it doesn't
   generate a profile about you.
 - SearXNG doesn't care about what you search for, never shares anything with a
-  third-party, and it can't be used to compromise you.
-- SearXNG is free software, the code is 100% open, and everyone is welcome to
+  third-party, and can't be used to compromise you.
+- SearXNG is free software; the code is 100% open, and everyone is welcome to
   make it better.
 
 If you do care about privacy, want to be a conscious user, or otherwise believe
@@ -41,12 +41,12 @@ search engine, see your browser's documentation:
 When adding a search engine, there must be no duplicates with the same name.  If
 you encounter a problem where you cannot add the search engine, you can either:
 
-- remove the duplicate (default name: SearXNG) or
-- contact the owner to give the instance a different name than the default.
+- Remove the duplicate (default name: SearXNG) or
+- Contact the owner to give the instance a different name from the default.
 
 ## How does it work?
 
-SearXNG is a fork from the well-known [searx] [metasearch engine] which was
+SearXNG is a fork of the well-known [searx] [metasearch engine] which was
 inspired by the [Seeks project].  It provides basic privacy by mixing your
 queries with searches on other platforms without storing search data.  SearXNG
 can be added to your browser's search bar; moreover, it can be set as the

searx/infopage/en/search-syntax.md

@@ -1,64 +1,64 @@
 # Search syntax
 
-SearXNG comes with a search syntax by with you can modify the categories,
-engines, languages and more.  See the {{link('preferences', 'preferences')}} for
-the list of engines, categories and languages.
+SearXNG comes with a search syntax by which you can modify the categories,
+engines, languages, and more.  See the {{link('preferences', 'preferences')}} for
+the list of engines, categories, and languages.
 
-## `!` select engine and category
+## `!` Select engine and category
 
-To set category and/or engine names use a `!` prefix.  To give a few examples:
+To set category and/or engine names, use a `!` prefix.  To give a few examples:
 
-- search in Wikipedia for **paris**
+- Search Wikipedia for **paris**:
 
   - {{search('!wp paris')}}
   - {{search('!wikipedia paris')}}
 
-- search in category **map** for **paris**
+- Search in category **map** for **paris**:
 
   - {{search('!map paris')}}
 
-- image search
+- Image search
 
   - {{search('!images Wau Holland')}}
 
 Abbreviations of the engines and languages are also accepted.  Engine/category
-modifiers are chain able and inclusive.  E.g. with {{search('!map !ddg !wp
-paris')}} search in map category and DuckDuckGo and Wikipedia for **paris**.
+modifiers are chainable and inclusive.  For example, {{search('!map !ddg !wp
+paris')}} searches in the map category and searches DuckDuckGo and Wikipedia for **paris**.
 
-## `:` select language
+## `:` Select language
 
-To select language filter use a `:` prefix.  To give an example:
+To select a language filter use a `:` prefix.  To give an example:
 
-- search Wikipedia by a custom language
+- Search Wikipedia with a custom language:
 
   - {{search(':fr !wp Wau Holland')}}
 
-## `!!<bang>` external bangs
+## `!!<bang>` External bangs
 
 SearXNG supports the external bangs from [DuckDuckGo].  To directly jump to a
 external search page use the `!!` prefix.  To give an example:
 
-- search Wikipedia by a custom language
+- Search Wikipedia with a custom language:
 
   - {{search('!!wfr Wau Holland')}}
 
-Please note, your search will be performed directly in the external search
-engine, SearXNG cannot protect your privacy on this.
+Please note that your search will be performed directly in the external search
+engine.  SearXNG cannot protect your privacy with this.
 
 [DuckDuckGo]: https://duckduckgo.com/bang
 
 ## `!!` automatic redirect
 
-When mentioning `!!` within the search query (separated by spaces), you will
+When including `!!` within your search query (separated by spaces), you will
 automatically be redirected to the first result.  This behavior is comparable to
 the "Feeling Lucky" feature from DuckDuckGo.  To give an example:
 
-- search for a query and get redirected to the first result
+- Search for a query and get redirected to the first result
 
   - {{search('!! Wau Holland')}}
 
-Please keep in mind that the result you are being redirected to can't become
-verified for being trustworthy, SearXNG cannot protect your personal privacy
+Please keep in mind that the result you are being redirected to can't be
+verified for trustworthiness and SearXNG cannot protect your personal privacy
 when using this feature.  Use it at your own risk.
 
 ## Special Queries
@@ -66,19 +66,19 @@ when using this feature.  Use it at your own risk.
 In the {{link('preferences', 'preferences')}} page you find keywords for
 _special queries_.  To give a few examples:
 
-- generate a random UUID
+- Generate a random UUID
 
   - {{search('random uuid')}}
 
-- find the average
+- Find the average
 
   - {{search('avg 123 548 2.04 24.2')}}
 
-- show _user agent_ of your browser (needs to be activated)
+- Show the _user agent_ of your browser (needs to be activated)
 
   - {{search('user-agent')}}
 
-- convert strings to different hash digests (needs to be activated)
+- Convert strings to different hash digests (needs to be activated)
 
   - {{search('md5 lorem ipsum')}}
   - {{search('sha512 lorem ipsum')}}

searx/plugins/hostnames.py

@@ -121,7 +121,7 @@ class SXNGPlugin(Plugin):
         self.info = PluginInfo(
             id=self.id,
             name=gettext("Hostnames plugin"),
-            description=gettext("Rewrite hostnames, remove results or prioritize them based on the hostname"),
+            description=gettext("Rewrite hostnames and remove or prioritize results based on the hostname"),
             preference_section="general",
         )
 

searx/plugins/tracker_url_remover.py

@@ -1,36 +1,34 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
-# pylint: disable=missing-module-docstring
+# pylint: disable=missing-module-docstring, unused-argument
 
 from __future__ import annotations
-import typing
 
-import re
-from urllib.parse import urlunparse, parse_qsl, urlencode
+import logging
+import typing
 
 from flask_babel import gettext
 
-from searx.plugins import Plugin, PluginInfo
+from searx.data import TRACKER_PATTERNS
+
+from . import Plugin, PluginInfo
 
 if typing.TYPE_CHECKING:
     from searx.search import SearchWithPlugins
     from searx.extended_types import SXNG_Request
-    from searx.result_types import Result
+    from searx.result_types import Result, LegacyResult
     from searx.plugins import PluginCfg
 
-regexes = {
-    re.compile(r'utm_[^&]+'),
-    re.compile(r'(wkey|wemail)[^&]*'),
-    re.compile(r'(_hsenc|_hsmi|hsCtaTracking|__hssc|__hstc|__hsfp)[^&]*'),
-    re.compile(r'&$'),
-}
+
+log = logging.getLogger("searx.plugins.tracker_url_remover")
 
 
 class SXNGPlugin(Plugin):
-    """Remove trackers arguments from the returned URL"""
+    """Remove trackers arguments from the returned URL."""
 
     id = "tracker_url_remover"
 
     def __init__(self, plg_cfg: "PluginCfg") -> None:
+
         super().__init__(plg_cfg)
         self.info = PluginInfo(
             id=self.id,
@@ -39,20 +37,18 @@ class SXNGPlugin(Plugin):
             preference_section="privacy",
         )
 
-    def on_result(
-        self, request: "SXNG_Request", search: "SearchWithPlugins", result: Result
-    ) -> bool:  # pylint: disable=unused-argument
-        if not result.parsed_url:
+    def on_result(self, request: "SXNG_Request", search: "SearchWithPlugins", result: Result) -> bool:
+
+        result.filter_urls(self.filter_url_field)
+        return True
+
+    @classmethod
+    def filter_url_field(cls, result: "Result|LegacyResult", field_name: str, url_src: str) -> bool | str:
+        """Returns bool ``True`` to use URL unchanged (``False`` to ignore URL).
+        If URL should be modified, the returned string is the new URL to use."""
+
+        if not url_src:
+            log.debug("missing a URL in field %s", field_name)
             return True
 
-        parsed_query: list[tuple[str, str]] = parse_qsl(result.parsed_url.query)
-        for name_value in list(parsed_query):
-            param_name = name_value[0]
-            for reg in regexes:
-                if reg.match(param_name):
-                    parsed_query.remove(name_value)
-                    result.parsed_url = result.parsed_url._replace(query=urlencode(parsed_query))
-                    result.url = urlunparse(result.parsed_url)
-                    break
-
-        return True
+        return TRACKER_PATTERNS.clean_url(url=url_src)

searx/settings.yml

@@ -245,7 +245,7 @@ plugins:
     active: false
 
   searx.plugins.tracker_url_remover.SXNGPlugin:
-    active: false
+    active: true
 
 
 # Configuration of the "Hostnames plugin":

searx/templates/simple/preferences/center_alignment.html

@@ -11,6 +11,6 @@
            >{{- '' -}}
   </p>{{- '' -}}
   <div class="description">
-    {{- _('Displays results in the center of the page (Oscar layout).') -}}
+    {{- _('Display results in the center of the page (Oscar layout).') -}}
   </div>{{- '' -}}
 </fieldset>{{- '' -}}

searx/templates/simple/preferences/cookies.html

@@ -1,6 +1,6 @@
 <p class="text-muted">
   {{- _('This is the list of cookies and their values SearXNG is storing on your computer.') }}
-  <br>{{- _('With that list, you can assess SearXNG transparency.') -}}
+  <br>{{- _('With this list, you can assess the transparency of SearXNG.') -}}
   <br>{{- '' -}}
 </p>
 {% if cookies %}

searx/templates/simple/preferences/footer.html

@@ -1,6 +1,6 @@
 <p class="small_font">
-  {{- _('These settings are stored in your cookies, this allows us not to store this data about you.') -}}
-  <br>{{- _("These cookies serve your sole convenience, we don't use these cookies to track you.") -}}
+  {{- _('These settings are stored in your cookies. This allows us not to store this data about you.') -}}
+  <br>{{- _("These cookies serve your sole convenience; we don't use them to track you.") -}}
 </p>{{- '' -}}
 
 <input type="submit" value="{{ _('Save') }}">{{- '' -}}

searx/templates/simple/preferences/image_proxy.html

@@ -11,6 +11,6 @@
            >{{- '' -}}
   </p>{{- '' -}}
   <div class="description">
-    {{- _('Proxying image results through SearXNG') -}}
+    {{- _('Proxy image results through SearXNG') -}}
   </div>{{- '' -}}
 </fieldset>{{- '' -}}

searx/templates/simple/preferences/infinite_scroll.html

@@ -11,6 +11,6 @@
            >{{- '' -}}
   </p>{{- '' -}}
   <div class="description" id="pref_infinite_scroll">
-    {{- _('Automatically load next page when scrolling to bottom of current page') -}}
+    {{- _('Automatically load the next page when scrolling to the bottom of the current page') -}}
   </div>{{- '' -}}
 </fieldset>{{- '' -}}

searx/templates/simple/preferences/search_on_category_select.html

@@ -11,6 +11,6 @@
            >{{- '' -}}
   </p>{{- '' -}}
   <div class="description" id="pref_search_on_category_select">
-    {{- _('Perform search immediately if a category selected. Disable to select multiple categories') -}}
+    {{- _('Perform a search immediately if a category is selected. Disable to select multiple categories') -}}
   </div>{{- '' -}}
-</fieldset>{{- '' -}}
+</fieldset>{{- '' -}}

searx/templates/simple/preferences/theme.html

@@ -11,7 +11,7 @@
     </select>{{- '' -}}
   </div>{{- '' -}}
   <div class="description">
-    {{- _('Change SearXNG layout') -}}
+    {{- _('Change the layout of SearXNG') -}}
   </div>{{- '' -}}
 </fieldset>{{- '' -}}