[fix] brave.news engine: response is HTML and no longer JSON

The response from brave.com for news is no longer a JSON string.

Closes: https://github.com/searxng/searxng/issues/4352
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

AUTHORS.rst

@@ -175,3 +175,4 @@ features or generally made searx better:
 - Daniel Kukula `<https://github.com/dkuku>`
 - Patrick Evans `https://github.com/holysoles`
 - Daniel Mowitz `<https://daniel.mowitz.rocks>`
+- `Bearz314 <https://github.com/bearz314>`_

searx/botdetection/ip_limit.py

@@ -123,7 +123,9 @@ def filter_request(
         )
         if c > SUSPICIOUS_IP_MAX:
             logger.error("BLOCK: too many request from %s in SUSPICIOUS_IP_WINDOW (redirect to /)", network)
-            return flask.redirect(flask.url_for('index'), code=302)
+            response = flask.redirect(flask.url_for('index'), code=302)
+            response.headers["Cache-Control"] = "no-store, max-age=0"
+            return response
 
         c = incr_sliding_window(redis_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
         if c > BURST_MAX_SUSPICIOUS:

searx/engines/brave.py

@@ -254,14 +254,14 @@ def response(resp) -> EngineResults:
     if brave_category in ('search', 'goggles'):
         return _parse_search(resp)
 
+    if brave_category in ('news'):
+        return _parse_news(resp)
+
     datastr = extr(resp.text, "const data = ", ";\n").strip()
 
     json_data = js_variable_to_python(datastr)
     json_resp = json_data[1]['data']['body']['response']
 
-    if brave_category == 'news':
-        return _parse_news(json_resp['news'])
-
     if brave_category == 'images':
         return _parse_images(json_resp)
     if brave_category == 'videos':
@@ -339,18 +339,31 @@ def _parse_search(resp) -> EngineResults:
     return result_list
 
 
-def _parse_news(json_resp) -> EngineResults:
-    result_list = EngineResults()
+def _parse_news(resp) -> EngineResults:
+
+    result_list = EngineResults()
+    dom = html.fromstring(resp.text)
+
+    for result in eval_xpath_list(dom, '//div[contains(@class, "results")]//div[@data-type="news"]'):
+
+        # import pdb
+        # pdb.set_trace()
+
+        url = eval_xpath_getindex(result, './/a[contains(@class, "result-header")]/@href', 0, default=None)
+        if url is None:
+            continue
+
+        title = extract_text(eval_xpath_list(result, './/span[contains(@class, "snippet-title")]'))
+        content = extract_text(eval_xpath_list(result, './/p[contains(@class, "desc")]'))
+        thumbnail = eval_xpath_getindex(result, './/div[contains(@class, "image-wrapper")]//img/@src', 0, default='')
 
-    for result in json_resp["results"]:
         item = {
-            'url': result['url'],
-            'title': result['title'],
-            'content': result['description'],
-            'publishedDate': _extract_published_date(result['age']),
+            "url": url,
+            "title": title,
+            "content": content,
+            "thumbnail": thumbnail,
         }
-        if result['thumbnail'] is not None:
-            item['thumbnail'] = result['thumbnail']['src']
+
         result_list.append(item)
 
     return result_list

searx/engines/mojeek.py

@@ -67,11 +67,13 @@ def request(query, params):
     args = {
         'q': query,
         'safe': min(params['safesearch'], 1),
-        'fmt': search_type,
         language_param: traits.get_language(params['searxng_locale'], traits.custom['language_all']),
         region_param: traits.get_region(params['searxng_locale'], traits.custom['region_all']),
     }
 
+    if search_type:
+        args['fmt'] = search_type
+
     if search_type == '':
         args['s'] = 10 * (params['pageno'] - 1)
 

searx/webapp.py

@@ -594,7 +594,7 @@ def health():
 @app.route('/client<token>.css', methods=['GET', 'POST'])
 def client_token(token=None):
     link_token.ping(sxng_request, token)
-    return Response('', mimetype='text/css')
+    return Response('', mimetype='text/css', headers={"Cache-Control": "no-store, max-age=0"})
 
 
 @app.route('/rss.xsl', methods=['GET', 'POST'])