Merge pull request #1743 from dalf/update_about_metrics

Update about the metrics

docs/admin/engines/settings.rst

@@ -240,6 +240,7 @@ Global Settings
      query_in_title: false
      infinite_scroll: false
      center_alignment: false
+     cache_url: https://web.archive.org/web/
      default_theme: simple
      theme_args:
        simple_style: auto
@@ -267,6 +268,15 @@ Global Settings
   side of the screen.  This setting only affects the *desktop layout*
   (:origin:`min-width: @tablet <searx/static/themes/simple/src/less/definitions.less>`)
 
+.. cache_url:
+
+``cache_url`` : ``https://web.archive.org/web/``
+  URL prefix of the internet archive or cache, don't forgett trailing slash (if
+  needed).  The default is https://web.archive.org/web/ alternatives are:
+
+  - https://webcache.googleusercontent.com/search?q=cache:
+  - https://archive.today/
+
 ``default_theme`` :
   Name of the theme you want to use by default on your SearXNG instance.
 

searx/engines/xpath.py

@@ -22,6 +22,7 @@ from urllib.parse import urlencode
 
 from lxml import html
 from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list
+from searx.network import raise_for_httperror
 
 search_url = None
 """
@@ -61,6 +62,14 @@ lang_all = 'en'
 selected.
 '''
 
+no_result_for_http_status = []
+'''Return empty result for these HTTP status codes instead of throwing an error.
+
+.. code:: yaml
+
+    no_result_for_http_status: []
+'''
+
 soft_max_redirects = 0
 '''Maximum redirects, soft limit. Record an error but don't stop the engine'''
 
@@ -177,11 +186,18 @@ def request(query, params):
     params['url'] = search_url.format(**fargs)
     params['soft_max_redirects'] = soft_max_redirects
 
+    params['raise_for_httperror'] = False
+
     return params
 
 
-def response(resp):
+def response(resp):  # pylint: disable=too-many-branches
     '''Scrap *results* from the response (see :ref:`engine results`).'''
+    if no_result_for_http_status and resp.status_code in no_result_for_http_status:
+        return []
+
+    raise_for_httperror(resp)
+
     results = []
     dom = html.fromstring(resp.text)
     is_onion = 'onions' in categories

searx/settings.yml

@@ -95,6 +95,8 @@ ui:
   default_theme: simple
   # center the results ?
   center_alignment: false
+  # URL prefix of the internet archive, don't forgett trailing slash (if needed).
+  # cache_url: "https://webcache.googleusercontent.com/search?q=cache:"
   # Default interface locale - leave blank to detect from browser information or
   # use codes from the 'locales' config section
   default_locale: ""
@@ -1808,6 +1810,7 @@ engines:
     url_xpath: //div[@class="upper-synonyms"]/a/@href
     content_xpath: //div[@class="synonyms-list-group"]
     title_xpath: //div[@class="upper-synonyms"]/a
+    no_result_for_http_status: [404]
     about:
       website: https://www.woxikon.de/
       wikidata_id:  # No Wikidata ID

searx/settings_defaults.py

@@ -190,6 +190,7 @@ SCHEMA = {
         'advanced_search': SettingsValue(bool, False),
         'query_in_title': SettingsValue(bool, False),
         'infinite_scroll': SettingsValue(bool, False),
+        'cache_url': SettingsValue(str, 'https://web.archive.org/web/'),
     },
     'preferences': {
         'lock': SettingsValue(list, []),

searx/templates/simple/base.html

@@ -62,9 +62,9 @@
   <footer>
     <p>
     {{ _('Powered by') }} <a href="{{ url_for('info', pagename='about') }}">searxng</a> - {{ searx_version }} — {{ _('a privacy-respecting, hackable metasearch engine') }}<br/>
-        <a href="{{ searx_git_url }}">{{ _('Source code') }}</a> |
+        <a href="{{ searx_git_url }}">{{ _('Source code') }}</a>
-        <a href="{{ get_setting('brand.issue_url') }}">{{ _('Issue tracker') }}</a> |
+        | <a href="{{ get_setting('brand.issue_url') }}">{{ _('Issue tracker') }}</a>
-        <a href="{{ url_for('stats') }}">{{ _('Engine stats') }}</a>
+        {% if enable_metrics %}| <a href="{{ url_for('stats') }}">{{ _('Engine stats') }}</a>{% endif %}
         {% if get_setting('brand.public_instances') %}
         | <a href="{{ get_setting('brand.public_instances') }}">{{ _('Public instances') }}</a>
         {% endif %}

searx/templates/simple/macros.html

@@ -42,7 +42,7 @@
 {%- macro result_sub_footer(result, proxify) -%}
 <div class="engines">
   {% for engine in result.engines %}<span>{{ engine }}</span>{% endfor %}
-  {{ result_link("https://web.archive.org/web/" + result.url, icon_small('ellipsis-vertical-outline') + _('cached'), "cache_link") }}&lrm; {% if proxify and proxify_results %} {{ result_link(proxify(result.url), icon('link') + _('proxied'), "proxyfied_link") }} {% endif %}
+  {{ result_link(cache_url + result.url, icon_small('ellipsis-vertical-outline') + _('cached'), "cache_link") }}&lrm; {% if proxify and proxify_results %} {{ result_link(proxify(result.url), icon('link') + _('proxied'), "proxyfied_link") }} {% endif %}
 </div>{{- '' -}}
 <div class="break"></div>{{- '' -}}
 {%- endmacro -%}

searx/templates/simple/preferences.html

@@ -310,9 +310,9 @@
         <th>{{ _("Supports selected language") }}</th>{{- "" -}}
         <th>{{ _("SafeSearch") }}</th>{{- "" -}}
         <th>{{ _("Time range") }}</th>{{- "" -}}
-        <th>{{ _("Response time") }}</th>{{- "" -}}
+        {%- if enable_metrics %}<th>{{ _("Response time") }}</th>{% endif -%}
         <th>{{ _("Max time") }}</th>{{- "" -}}
-        <th>{{ _("Reliability") }}</th>{{- "" -}}
+        {%- if enable_metrics %}<th>{{ _("Reliability") }}</th>{% endif -%}
       </tr>
       {% for group, engines in engines_by_category[categ] | group_engines_in_tab %}
       {% if loop.length > 1 %}
@@ -336,9 +336,9 @@
         <td>{{ checkbox(None, supports[search_engine.name]['supports_selected_language'], true) }}</td>{{- "" -}}
         <td>{{ checkbox(None, supports[search_engine.name]['safesearch'], true) }}</td>{{- "" -}}
         <td>{{ checkbox(None, supports[search_engine.name]['time_range_support'], true) }}</td>{{- "" -}}
-        {{- engine_time(search_engine.name) -}}
+        {%- if enable_metrics %}{{- engine_time(search_engine.name) -}}{% endif -%}
         <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td>{{- "" -}}
-        {{ engine_reliability(search_engine.name) -}}
+        {%- if enable_metrics %}{{ engine_reliability(search_engine.name) -}}{% endif -%}
       </tr>
       {% endif %}
       {% endfor %}

searx/templates/simple/stats.html

@@ -33,10 +33,7 @@
         <td class="engine-name"><a href="{{ url_for('stats', engine=engine_stat.name|e) }}">{{ engine_stat.name }}</a></td>
         <td class="engine-score">
             {% if engine_stat.score %}
-            <span aria-labelledby="{{engine_stat.name}}_score" >{{ engine_stat.score|round(1) }}</span>
+            <span>{{ engine_stat.score_per_result|round(1) }}</span>
-            <div class="engine-tooltip" role="tooltip" id="{{engine_stat.name}}_score">{{- "" -}}
-                <p>{{ _('Scores per result') }}: {{ engine_stat.score_per_result | round(3) }}</p>
-            </div>
             {% endif %}
         </td>
         <td class="engine-result-count">

searx/webapp.py

@@ -145,7 +145,7 @@ result_templates = get_result_templates(templates_path)
 
 STATS_SORT_PARAMETERS = {
     'name': (False, 'name', ''),
-    'score': (True, 'score', 0),
+    'score': (True, 'score_per_result', 0),
     'result_count': (True, 'result_count', 0),
     'time': (False, 'total', 0),
     'reliability': (False, 'reliability', 100),
@@ -450,6 +450,7 @@ def render(template_name: str, **kwargs):
     kwargs['instance_name'] = get_setting('general.instance_name')
     kwargs['searx_version'] = VERSION_STRING
     kwargs['searx_git_url'] = GIT_URL
+    kwargs['enable_metrics'] = get_setting('general.enable_metrics')
     kwargs['get_setting'] = get_setting
     kwargs['get_pretty_url'] = get_pretty_url
 
@@ -464,6 +465,7 @@ def render(template_name: str, **kwargs):
     kwargs['image_proxify'] = image_proxify
     kwargs['proxify'] = morty_proxify if settings['result_proxy']['url'] is not None else None
     kwargs['proxify_results'] = settings['result_proxy']['proxify_results']
+    kwargs['cache_url'] = settings['ui']['cache_url']
     kwargs['get_result_template'] = get_result_template
     kwargs['opensearch_url'] = (
         url_for('opensearch')