Merge 5e11d697ab into 10d3af84b8

The query string send to DDG must not be qouted.

The query string was URL-qouted in #4011, but the URL-qouted query string result
in unexpected *URL decoded* and other garbish results as reported in #4019
and #4020.  To test compare the results of a query like::

    !ddg Häuser und Straßen :de
    !ddg Häuser und Straßen :all
    !ddg 房屋和街道 :all
    !ddg 房屋和街道 :zh

Closed:

- [#4019] https://github.com/searxng/searxng/issues/4019
- [#4020] https://github.com/searxng/searxng/issues/4020

Related:

- [#4011] https://github.com/searxng/searxng/pull/4011

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/engines/duckduckgo.py

@@ -6,7 +6,7 @@ DuckDuckGo Lite
 
 from typing import TYPE_CHECKING
 import re
-from urllib.parse import urlencode, quote_plus
+from urllib.parse import urlencode
 import json
 import babel
 import lxml.html
@@ -263,7 +263,7 @@ def request(query, params):
 
     params['url'] = url
     params['method'] = 'POST'
-    params['data']['q'] = quote_plus(query)
+    params['data']['q'] = query
 
     # The API is not documented, so we do some reverse engineering and emulate
     # what https://html.duckduckgo.com/html does when you press "next Page" link
@@ -381,7 +381,11 @@ def response(resp):
     zero_click_info_xpath = '//div[@id="zero_click_abstract"]'
     zero_click = extract_text(eval_xpath(doc, zero_click_info_xpath)).strip()
 
-    if zero_click and "Your IP address is" not in zero_click and "Your user agent:" not in zero_click:
+    if zero_click and (
+        "Your IP address is" not in zero_click
+        and "Your user agent:" not in zero_click
+        and "URL Decoded:" not in zero_click
+    ):
         current_query = resp.search_params["data"].get("q")
 
         results.append(

searx/engines/google.py

@@ -320,6 +320,51 @@ def _parse_data_images(dom):
     return data_image_map
 
 
+def _eval_answers(results, dom, xpath):
+    answer_list = eval_xpath(dom, xpath)
+    drop_elements = [
+        './/div[@class="nnFGuf"]',
+        './/script',  # Scripts like the calculator
+        './/table[@class="HOoTuc"]',  # The actual calculator controls
+        './/table[@class="ElumCf"]',  # The actual calculator buttons
+        './/div[@class="mDRaHd"]',  # Instructions with links
+        './/span[@class="W7GCoc CNbPnc"]',  # Feedback
+        './/*[@style="display:none"]',  # Hidden elements
+    ]
+    for item in answer_list:
+        for element in eval_xpath(item, ' | '.join(drop_elements)):
+            element.drop_tree()
+        extracted_table_html = None
+        is_safe = False
+        table_elements = eval_xpath(item, './/table')
+        if table_elements:
+            extracted_table = table_elements[0]
+            extracted_table.attrib.clear()
+            for element in extracted_table.xpath('.//*'):
+                element.attrib.clear()
+            extracted_table.set('cellpadding', '2')
+            extracted_table.set('cellspacing', '2')
+            extracted_table.set('border', '0')
+            extracted_table_html = html.tostring(extracted_table, pretty_print=True, encoding='unicode')
+            is_safe = True
+        for element in eval_xpath(item, './/table'):  # Drop all remaining tables
+            element.drop_tree()
+        answer_content = extract_text(item)
+        if extracted_table_html:
+            answer_content += '<p>' + extracted_table_html
+        url = (eval_xpath(item, '../..//a/@href') + [None])[0]
+        if url and url.startswith('/search?'):  # If the answer is a Google search link, don't use it
+            url = None
+        results.append(
+            {
+                'answer': answer_content,
+                'url': url,
+                'safe': is_safe,
+            }
+        )
+    return results
+
+
 def response(resp):
     """Get response from google's search request"""
     # pylint: disable=too-many-branches, too-many-statements
@@ -331,17 +376,21 @@ def response(resp):
     dom = html.fromstring(resp.text)
     data_image_map = _parse_data_images(dom)
 
-    # results --> answer
+    results = _eval_answers(results, dom, '//div[contains(@class, "card-section")]')  # Look for cards first
-    answer_list = eval_xpath(dom, '//div[contains(@class, "LGOjhe")]')
+    if not results:
-    for item in answer_list:
+        results = _eval_answers(results, dom, '//div[contains(@class, "LGOjhe")]')  # Look for rendered answers next
-        for bubble in eval_xpath(item, './/div[@class="nnFGuf"]'):
+    # Look for JS DOM encoded string answers last
-            bubble.drop_tree()
+    if not results:
-        results.append(
+        pattern = r"'\\x3c.*?\\x3e'"  # These are DOM encoded strings that Google will push into the DOM
-            {
+        matches = re.findall(pattern, resp.text)
-                'answer': extract_text(item),
+        for match in matches:
-                'url': (eval_xpath(item, '../..//a/@href') + [None])[0],
+            decoded_html = match.encode().decode('unicode_escape')
-            }
+            encoded_dom = html.fromstring(decoded_html)
-        )
+            sub_doms = eval_xpath(encoded_dom, '//div[contains(@class, "LGOjhe")]')
+            if sub_doms:
+                if '<span class="hgKElc"><b>' in decoded_html:  # Main answers start with a bold
+                    results = _eval_answers(results, encoded_dom, '//div[contains(@class, "LGOjhe")]')
+                break  # If it's a JS encoded answer, we only want the first one if it has bold above
 
     # parse results
 

searx/templates/simple/results.html

@@ -23,7 +23,11 @@
     <div id="answers" role="complementary" aria-labelledby="answers-title"><h4 class="title" id="answers-title">{{ _('Answers') }} : </h4>
         {%- for answer in answers.values() -%}
         <div class="answer">
+          {%- if answer.safe -%}
+            <span>{{ answer.answer | safe }}</span>
+          {%- else -%}
             <span>{{ answer.answer }}</span>
+          {%- endif -%}
           {%- if answer.url -%}
           <a href="{{ answer.url }}" class="answer-url"
              {%- if results_on_new_tab %} target="_blank" rel="noopener noreferrer"