Merge pull request #2373 from kvch/display-https-engines

Display if an engine does not support HTTPS requests

searx/engines/__init__.py

@@ -20,6 +20,7 @@ import sys
 import threading
 from os.path import realpath, dirname
 from babel.localedata import locale_identifiers
+from urllib.parse import urlparse
 from flask_babel import gettext
 from operator import itemgetter
 from searx import settings
@@ -289,3 +290,34 @@ def initialize_engines(engine_list):
             if init_fn:
                 logger.debug('%s engine: Starting background initialization', engine_name)
                 threading.Thread(target=engine_init, args=(engine_name, init_fn)).start()
+
+        _set_https_support_for_engine(engine)
+
+
+def _set_https_support_for_engine(engine):
+    # check HTTPS support if it is not disabled
+    if not engine.offline and not hasattr(engine, 'https_support'):
+        params = engine.request('http_test', {
+            'method': 'GET',
+            'headers': {},
+            'data': {},
+            'url': '',
+            'cookies': {},
+            'verify': True,
+            'auth': None,
+            'pageno': 1,
+            'time_range': None,
+            'language': '',
+            'safesearch': False,
+            'is_test': True,
+            'category': 'files',
+            'raise_for_status': True,
+        })
+
+        if 'url' not in params:
+            return
+
+        parsed_url = urlparse(params['url'])
+        https_support = parsed_url.scheme == 'https'
+
+        setattr(engine, 'https_support', https_support)

searx/engines/acgsou.py

@@ -18,7 +18,7 @@ categories = ['files', 'images', 'videos', 'music']
 paging = True
 
 # search-url
-base_url = 'http://www.acgsou.com/'
+base_url = 'https://www.acgsou.com/'
 search_url = base_url + 'search.php?{query}&page={offset}'
 # xpath queries
 xpath_results = '//table[contains(@class, "list_style table_fixed")]//tr[not(th)]'
@@ -40,7 +40,7 @@ def response(resp):
     for result in eval_xpath_list(dom, xpath_results):
         # defaults
         filesize = 0
-        magnet_link = "magnet:?xt=urn:btih:{}&tr=http://tracker.acgsou.com:2710/announce"
+        magnet_link = "magnet:?xt=urn:btih:{}&tr=https://tracker.acgsou.com:2710/announce"
 
         category = extract_text(eval_xpath_getindex(result, xpath_category, 0, default=[]))
         page_a = eval_xpath_getindex(result, xpath_title, 0)

searx/engines/arxiv.py

@@ -19,7 +19,7 @@ from searx.utils import eval_xpath_list, eval_xpath_getindex
 categories = ['science']
 paging = True
 
-base_url = 'http://export.arxiv.org/api/query?search_query=all:'\
+base_url = 'https://export.arxiv.org/api/query?search_query=all:'\
            + '{query}&start={offset}&max_results={number_of_results}'
 
 # engine dependent config

searx/engines/currency_convert.py

@@ -9,6 +9,7 @@ url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'
 weight = 100
 
 parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
+https_support = True
 
 
 def normalize_name(name):

searx/engines/dictzone.py

@@ -20,6 +20,7 @@ weight = 100
 
 parser_re = re.compile('.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
 results_xpath = './/table[@id="r"]/tr'
+https_support = True
 
 
 def request(query, params):

searx/engines/translated.py

@@ -15,6 +15,7 @@ categories = ['general']
 url = 'https://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'
 web_url = 'https://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'
 weight = 100
+https_support = True
 
 parser_re = re.compile('.*?([a-z]+)-([a-z]+) (.{2,})$', re.I)
 api_key = ''

searx/templates/oscar/macros.html

@@ -1,6 +1,6 @@
 <!-- Draw glyphicon icon from bootstrap-theme -->
-{% macro icon(action) -%}
-    <span class="glyphicon glyphicon-{{ action }}"></span>
+{% macro icon(action, alt) -%}
+    <span title="{{ alt }}" class="glyphicon glyphicon-{{ action }}"></span>
 {%- endmacro %}
 
 <!-- Draw favicon -->

searx/templates/oscar/preferences.html

@@ -230,8 +230,8 @@
                                     <td class="onoff-checkbox">
                                         {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in disabled_engines) }}
                                     </td>
-                                    <th scope="row">{{ search_engine.name }}</th>
-                                    <td class="name">{{ shortcuts[search_engine.name] }}</td>
+                                    <th scope="row">{% if not search_engine.https_support %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</td></th>
+                                    <td class="name">{{ shortcuts[search_engine.name] }}
                                         <td>{{ support_toggle(stats[search_engine.name].supports_selected_language) }}</td>
                                         <td>{{ support_toggle(search_engine.safesearch==True) }}</td>
                                         <td>{{ support_toggle(search_engine.time_range_support==True) }}</td>

searx/templates/simple/macros.html

@@ -1,6 +1,6 @@
 <!-- Draw glyphicon icon from bootstrap-theme -->
-{% macro icon(action) -%}
-    <span class="ion-icon-big ion-{{ action }}"></span>
+{% macro icon(action, alt) -%}
+    <span title="{{ alt }}" class="ion-icon-big ion-{{ action }}"></span>
 {%- endmacro %}
 
 {% macro icon_small(action) -%}

searx/templates/simple/preferences.html

@@ -1,4 +1,4 @@
-{% from 'simple/macros.html' import tabs_open, tabs_close, tab_header, tab_footer, checkbox_onoff, checkbox %}
+{% from 'simple/macros.html' import icon, tabs_open, tabs_close, tab_header, tab_footer, checkbox_onoff, checkbox %}
 
 {% extends "simple/base.html" %}
 
@@ -121,7 +121,7 @@
       {% set engine_id = 'engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_') %}
       <tr>
         <td class="engine_checkbox">{{ checkbox_onoff(engine_id, (search_engine.name, categ) in disabled_engines) }}</td>
-        <th class="name">{{ search_engine.name }}</th>
+        <th class="name">{% if not search_engine.https_support %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</th>
         <td class="shortcut">{{ shortcuts[search_engine.name] }}</td>
         <td>{{ checkbox(engine_id + '_supported_languages', current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages, true, true) }}</td>
         <td>{{ checkbox(engine_id + '_safesearch', search_engine.safesearch==True, true, true) }}</td>