Merge 6ed37fc9eb into 10d3af84b8

The query string send to DDG must not be qouted.

The query string was URL-qouted in #4011, but the URL-qouted query string result
in unexpected *URL decoded* and other garbish results as reported in #4019
and #4020.  To test compare the results of a query like::

    !ddg Häuser und Straßen :de
    !ddg Häuser und Straßen :all
    !ddg 房屋和街道 :all
    !ddg 房屋和街道 :zh

Closed:

- [#4019] https://github.com/searxng/searxng/issues/4019
- [#4020] https://github.com/searxng/searxng/issues/4020

Related:

- [#4011] https://github.com/searxng/searxng/pull/4011

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/duckduckgo.py

@@ -6,7 +6,7 @@ DuckDuckGo Lite
 
 from typing import TYPE_CHECKING
 import re
-from urllib.parse import urlencode, quote_plus
+from urllib.parse import urlencode
 import json
 import babel
 import lxml.html
@@ -263,7 +263,7 @@ def request(query, params):
 
     params['url'] = url
     params['method'] = 'POST'
-    params['data']['q'] = quote_plus(query)
+    params['data']['q'] = query
 
     # The API is not documented, so we do some reverse engineering and emulate
     # what https://html.duckduckgo.com/html does when you press "next Page" link
@@ -381,7 +381,11 @@ def response(resp):
     zero_click_info_xpath = '//div[@id="zero_click_abstract"]'
     zero_click = extract_text(eval_xpath(doc, zero_click_info_xpath)).strip()
 
-    if zero_click and "Your IP address is" not in zero_click and "Your user agent:" not in zero_click:
+    if zero_click and (
+        "Your IP address is" not in zero_click
+        and "Your user agent:" not in zero_click
+        and "URL Decoded:" not in zero_click
+    ):
         current_query = resp.search_params["data"].get("q")
 
         results.append(

searx/preferences.py

@@ -479,7 +479,6 @@ class Preferences:
         self.plugins = PluginsSetting('plugins', plugins=plugins)
         self.tokens = SetSetting('tokens')
         self.client = client or ClientPref()
-        self.unknown_params: Dict[str, str] = {}
 
     def get_as_url_params(self):
         """Return preferences as URL parameters"""
@@ -523,10 +522,6 @@ class Preferences:
                 self.plugins.parse_cookie(input_data.get('disabled_plugins', ''), input_data.get('enabled_plugins', ''))
             elif user_setting_name == 'tokens':
                 self.tokens.parse(user_setting)
-            elif not any(
-                user_setting_name.startswith(x) for x in ['enabled_', 'disabled_', 'engine_', 'category_', 'plugin_']
-            ):
-                self.unknown_params[user_setting_name] = user_setting
 
     def parse_form(self, input_data: Dict[str, str]):
         """Parse formular (``<input>``) data from a ``flask.request.form``"""
@@ -551,8 +546,7 @@ class Preferences:
                 disabled_plugins.append(user_setting_name)
             elif user_setting_name == 'tokens':
                 self.tokens.parse_form(user_setting)
-            else:
-                self.unknown_params[user_setting_name] = user_setting
+
         self.key_value_settings['categories'].parse_form(enabled_categories)
         self.engines.parse_form(disabled_engines)
         self.plugins.parse_form(disabled_plugins)
@@ -563,8 +557,6 @@ class Preferences:
         ret_val = None
         if user_setting_name in self.key_value_settings:
             ret_val = self.key_value_settings[user_setting_name].get_value()
-        if user_setting_name in self.unknown_params:
-            ret_val = self.unknown_params[user_setting_name]
         return ret_val
 
     def save(self, resp: flask.Response):
@@ -577,8 +569,6 @@ class Preferences:
         self.engines.save(resp)
         self.plugins.save(resp)
         self.tokens.save('tokens', resp)
-        for k, v in self.unknown_params.items():
-            resp.set_cookie(k, v, max_age=COOKIE_MAX_AGE)
         return resp
 
     def validate_token(self, engine):

tests/unit/test_preferences.py

@@ -1,10 +1,13 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # pylint: disable=missing-module-docstring, invalid-name
 
+import flask
+from mock import Mock
 from tests import SearxTestCase
 from searx import favicons
 from searx.locales import locales_initialize
 from searx.preferences import (
+    Setting,
     EnumStringSetting,
     MapSetting,
     SearchLanguageSetting,
@@ -13,6 +16,7 @@ from searx.preferences import (
     ValidationException,
 )
 from searx.plugins import Plugin
+from searx.preferences import Preferences
 
 locales_initialize()
 favicons.init()
@@ -125,10 +129,10 @@ class TestSettings(SearxTestCase):  # pylint: disable=missing-class-docstring
 
 
 class TestPreferences(SearxTestCase):  # pylint: disable=missing-class-docstring
-    def test_encode(self):
-        from searx.preferences import Preferences  # pylint: disable=import-outside-toplevel
+    def setUp(self):
+        self.preferences = Preferences(['simple'], ['general'], {}, [])
 
-        pref = Preferences(['simple'], ['general'], {}, [])
+    def test_encode(self):
         url_params = (
             'eJx1Vk1z4zYM_TXxRZNMd7eddg8-pe21nWnvGoiEJEQkofDDtvzrC1qSRdnbQxQTBA'
             'Hw8eGRCiJ27AnDsUOHHszBgOsSdHjU-Pr7HwfDCkweHCBFVmxHgxGPB7LiU4-eL9Px'
@@ -155,8 +159,48 @@ class TestPreferences(SearxTestCase):  # pylint: disable=missing-class-docstring
             'd-DZy7PtaVp2WgvPBpzCXUL_J1OGex48RVmOXzBU8_N3kqekkefRDzxNK2_Klp9mBJ'
             'wsUnXyRqq1mScHuYalUY7_AZTCR4s=&q='
         )
-        pref.parse_encoded_data(url_params)
+        self.preferences.parse_encoded_data(url_params)
         self.assertEqual(
-            vars(pref.key_value_settings['categories']),
+            vars(self.preferences.key_value_settings['categories']),
             {'value': ['general'], 'locked': False, 'choices': ['general', 'none']},
         )
+
+    def test_save_key_value_setting(self):
+        setting_key = 'foo'
+        setting_value = 'bar'
+
+        cookie_callback = {}
+
+        def set_cookie_callback(name, value, max_age):  # pylint: disable=unused-argument
+            cookie_callback[name] = value
+
+        response_mock = Mock(flask.Response)
+        response_mock.set_cookie = set_cookie_callback
+        self.preferences.key_value_settings = {
+            setting_key: Setting(
+                setting_value,
+                locked=False,
+            ),
+        }
+        self.preferences.save(response_mock)
+        self.assertIn(setting_key, cookie_callback)
+        self.assertEqual(cookie_callback[setting_key], setting_value)
+
+    def test_false_key_value_setting(self):
+        setting_key = 'foo'
+
+        cookie_callback = {}
+
+        def set_cookie_callback(name, value, max_age):  # pylint: disable=unused-argument
+            cookie_callback[name] = value
+
+        response_mock = Mock(flask.Response)
+        response_mock.set_cookie = set_cookie_callback
+        self.preferences.key_value_settings = {
+            setting_key: Setting(
+                '',
+                locked=True,
+            ),
+        }
+        self.preferences.save(response_mock)
+        self.assertNotIn(setting_key, cookie_callback)