Merge branch 'master' into engines/unsplash

tests/unit/engines/test_archlinux.py

@@ -4,7 +4,7 @@ from searx.engines import archlinux
 from searx.testing import SearxTestCase
 
 domains = {
-    'en': 'https://wiki.archlinux.org',
+    'all': 'https://wiki.archlinux.org',
     'de': 'https://wiki.archlinux.de',
     'fr': 'https://wiki.archlinux.fr',
     'ja': 'https://wiki.archlinuxjp.org',

tests/unit/engines/test_bing.py

@@ -18,6 +18,10 @@ class TestBingEngine(SearxTestCase):
         self.assertTrue('language%3AFR' in params['url'])
         self.assertTrue('bing.com' in params['url'])
 
+        dicto['language'] = 'all'
+        params = bing.request(query.encode('utf-8'), dicto)
+        self.assertTrue('language' in params['url'])
+
     def test_response(self):
         self.assertRaises(AttributeError, bing.response, None)
         self.assertRaises(AttributeError, bing.response, [])

tests/unit/engines/test_bing_images.py

@@ -9,6 +9,7 @@ class TestBingImagesEngine(SearxTestCase):
 
     def test_request(self):
         bing_images.supported_languages = ['fr-FR', 'en-US']
+        bing_images.language_aliases = {}
         query = 'test_query'
         dicto = defaultdict(dict)
         dicto['pageno'] = 1
@@ -52,7 +53,7 @@ class TestBingImagesEngine(SearxTestCase):
                 <li>
                     <div>
                         <div class="imgpt">
-                            <a m='{"purl":"page_url","murl":"img_url"}' mad='{"turl":"thumb_url"}'>
+                            <a m='{"purl":"page_url","murl":"img_url","turl":"thumb_url"}'>
                                 <img src="" alt="alt text" />
                             </a>
                         </div>
@@ -60,7 +61,7 @@ class TestBingImagesEngine(SearxTestCase):
                     </div>
                     <div>
                         <div class="imgpt">
-                            <a m='{"purl":"page_url2","murl":"img_url2"}' mad='{"turl":"thumb_url2"}'>
+                            <a m='{"purl":"page_url2","murl":"img_url2","turl":"thumb_url2"}'>
                                 <img src="" alt="alt text 2" />
                             </a>
                         </div>
@@ -71,7 +72,7 @@ class TestBingImagesEngine(SearxTestCase):
                 <li>
                     <div>
                         <div class="imgpt">
-                            <a m='{"purl":"page_url3","murl":"img_url3"}' mad='{"turl":"thumb_url3"}'>
+                            <a m='{"purl":"page_url3","murl":"img_url3","turl":"thumb_url3"}'>
                                 <img src="" alt="alt text 3" />
                             </a>
                         </div>

tests/unit/engines/test_bing_news.py

@@ -20,6 +20,10 @@ class TestBingNewsEngine(SearxTestCase):
         self.assertIn('bing.com', params['url'])
         self.assertIn('fr', params['url'])
 
+        dicto['language'] = 'all'
+        params = bing_news.request(query, dicto)
+        self.assertIn('en', params['url'])
+
     def test_no_url_in_request_year_time_range(self):
         dicto = defaultdict(dict)
         query = 'test_query'

tests/unit/engines/test_bing_videos.py

@@ -9,6 +9,7 @@ class TestBingVideosEngine(SearxTestCase):
 
     def test_request(self):
         bing_videos.supported_languages = ['fr-FR', 'en-US']
+        bing_videos.language_aliases = {}
         query = 'test_query'
         dicto = defaultdict(dict)
         dicto['pageno'] = 1

tests/unit/engines/test_dailymotion.py

@@ -19,6 +19,10 @@ class TestDailymotionEngine(SearxTestCase):
         self.assertTrue('dailymotion.com' in params['url'])
         self.assertTrue('fr' in params['url'])
 
+        dicto['language'] = 'all'
+        params = dailymotion.request(query, dicto)
+        self.assertTrue('en' in params['url'])
+
     def test_response(self):
         self.assertRaises(AttributeError, dailymotion.response, None)
         self.assertRaises(AttributeError, dailymotion.response, [])

tests/unit/engines/test_duckduckgo_images.py

@@ -14,7 +14,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase):
         dicto['is_test'] = True
         dicto['pageno'] = 1
         dicto['safesearch'] = 0
-        dicto['language'] = 'en-US'
+        dicto['language'] = 'all'
         params = duckduckgo_images.request(query, dicto)
         self.assertIn('url', params)
         self.assertIn(query, params['url'])

tests/unit/engines/test_faroo.py

@@ -20,6 +20,10 @@ class TestFarooEngine(SearxTestCase):
         self.assertIn('en', params['url'])
         self.assertIn('web', params['url'])
 
+        dicto['language'] = 'all'
+        params = faroo.request(query, dicto)
+        self.assertIn('en', params['url'])
+
         dicto['language'] = 'de-DE'
         params = faroo.request(query, dicto)
         self.assertIn('de', params['url'])

tests/unit/engines/test_gigablast.py

@@ -11,11 +11,16 @@ class TestGigablastEngine(SearxTestCase):
         dicto = defaultdict(dict)
         dicto['pageno'] = 0
         dicto['safesearch'] = 0
-        dicto['language'] = 'en-US'
+        dicto['language'] = 'all'
         params = gigablast.request(query, dicto)
         self.assertTrue('url' in params)
         self.assertTrue(query in params['url'])
         self.assertTrue('gigablast.com' in params['url'])
+        self.assertTrue('xx' in params['url'])
+
+        dicto['language'] = 'en-US'
+        params = gigablast.request(query, dicto)
+        self.assertTrue('en' in params['url'])
         self.assertFalse('en-US' in params['url'])
 
     def test_response(self):

tests/unit/engines/test_google.py

@@ -15,7 +15,8 @@ class TestGoogleEngine(SearxTestCase):
         return response
 
     def test_request(self):
-        google.supported_languages = ['en', 'fr', 'zh-CN']
+        google.supported_languages = ['en', 'fr', 'zh-CN', 'iw']
+        google.language_aliases = {'he': 'iw'}
 
         query = 'test_query'
         dicto = defaultdict(dict)
@@ -41,6 +42,12 @@ class TestGoogleEngine(SearxTestCase):
         self.assertIn('zh-CN', params['url'])
         self.assertIn('zh-CN', params['headers']['Accept-Language'])
 
+        dicto['language'] = 'he'
+        params = google.request(query, dicto)
+        self.assertIn('google.com', params['url'])
+        self.assertIn('iw', params['url'])
+        self.assertIn('iw', params['headers']['Accept-Language'])
+
     def test_response(self):
         self.assertRaises(AttributeError, google.response, None)
         self.assertRaises(AttributeError, google.response, [])
@@ -198,29 +205,13 @@ class TestGoogleEngine(SearxTestCase):
         html = u"""
         <html>
             <body>
-                <table>
-                    <tbody>
-                        <tr>
-                            <td>
-                                <font>
-                                    <label>
-                                        <span id="ten">English</span>
-                                    </label>
-                                </font>
-                            </td>
-                            <td>
-                                <font>
-                                    <label>
-                                        <span id="tzh-CN">中文 (简体)</span>
-                                    </label>
-                                    <label>
-                                        <span id="tzh-TW">中文 (繁體)</span>
-                                    </label>
-                                </font>
-                            </td>
-                        </tr>
-                    </tbody>
-                </table>
+                <div id="langSec">
+                    <div>
+                        <input name="lr" data-name="english" value="lang_en" />
+                        <input name="lr" data-name="中文 (简体)" value="lang_zh-CN" />
+                        <input name="lr" data-name="中文 (繁體)" value="lang_zh-TW" />
+                    </div>
+                </div>
             </body>
         </html>
         """

tests/unit/engines/test_google_news.py

@@ -10,6 +10,7 @@ class TestGoogleNewsEngine(SearxTestCase):
 
     def test_request(self):
         google_news.supported_languages = ['en-US', 'fr-FR']
+        google_news.language_aliases = {}
         query = 'test_query'
         dicto = defaultdict(dict)
         dicto['pageno'] = 1
@@ -20,6 +21,11 @@ class TestGoogleNewsEngine(SearxTestCase):
         self.assertIn(query, params['url'])
         self.assertIn('fr', params['url'])
 
+        dicto['language'] = 'all'
+        params = google_news.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertNotIn('fr', params['url'])
+
     def test_response(self):
         self.assertRaises(AttributeError, google_news.response, None)
         self.assertRaises(AttributeError, google_news.response, [])

tests/unit/engines/test_google_videos.py

@@ -30,16 +30,34 @@ class TestGoogleVideosEngine(SearxTestCase):
         <div>
             <div>
                 <div class="g">
-                    <div>
-                        <h3><a href="url_1">Title 1</h3>
+                    <div class="r">
+                        <a href="url_1"><h3>Title 1</h3></a>
+                    </div>
+                    <div class="s">
+                        <div>
+                            <a>
+                                <g-img>
+                                    <img id="vidthumb1">
+                                </g-img>
+                            </a>
+                        </div>
                     </div>
                     <div>
                         <span class="st">Content 1</span>
                     </div>
                 </div>
                 <div class="g">
-                    <div>
-                        <h3><a href="url_2">Title 2</h3>
+                    <div class="r">
+                        <a href="url_2"><h3>Title 2</h3></a>
+                    </div>
+                    <div class="s">
+                        <div>
+                            <a>
+                                <g-img>
+                                    <img id="vidthumb2">
+                                </g-img>
+                            </a>
+                        </div>
                     </div>
                     <div>
                         <span class="st">Content 2</span>
@@ -47,6 +65,7 @@ class TestGoogleVideosEngine(SearxTestCase):
                 </div>
             </div>
         </div>
+        <script>function _setImagesSrc(c,d,e){}</script>
         """
         response = mock.Mock(text=html)
         results = google_videos.response(response)

tests/unit/engines/test_mediawiki.py

@@ -18,6 +18,10 @@ class TestMediawikiEngine(SearxTestCase):
         self.assertIn('wikipedia.org', params['url'])
         self.assertIn('fr', params['url'])
 
+        dicto['language'] = 'all'
+        params = mediawiki.request(query, dicto)
+        self.assertIn('en', params['url'])
+
         mediawiki.base_url = "http://test.url/"
         mediawiki.search_url = mediawiki.base_url +\
                                  'w/api.php?action=query'\

tests/unit/engines/test_qwant.py

@@ -8,6 +8,7 @@ class TestQwantEngine(SearxTestCase):
 
     def test_request(self):
         qwant.supported_languages = ['en-US', 'fr-CA', 'fr-FR']
+        qwant.language_aliases = {}
         query = 'test_query'
         dicto = defaultdict(dict)
         dicto['pageno'] = 0
@@ -20,11 +21,10 @@ class TestQwantEngine(SearxTestCase):
         self.assertIn('qwant.com', params['url'])
         self.assertIn('fr_fr', params['url'])
 
-        dicto['language'] = 'en-US'
+        dicto['language'] = 'all'
         qwant.categories = ['news']
         params = qwant.request(query, dicto)
         self.assertFalse('fr' in params['url'])
-        self.assertIn('en_us', params['url'])
         self.assertIn('news', params['url'])
 
         dicto['language'] = 'fr'

tests/unit/engines/test_startpage.py

@@ -21,6 +21,10 @@ class TestStartpageEngine(SearxTestCase):
         self.assertIn('with_language', params['data'])
         self.assertIn('lang_fr', params['data']['with_language'])
 
+        dicto['language'] = 'all'
+        params = startpage.request(query, dicto)
+        self.assertNotIn('with_language', params['data'])
+
     def test_response(self):
         self.assertRaises(AttributeError, startpage.response, None)
         self.assertRaises(AttributeError, startpage.response, [])
@@ -31,14 +35,14 @@ class TestStartpageEngine(SearxTestCase):
         self.assertEqual(startpage.response(response), [])
 
         html = """
-        <div class='result' style=' *width : auto; *margin-right : 10%;'>
+        <li class="search-result search-item">
             <h3>
                 <a href='http://this.should.be.the.link/' id='title_2' name='title_2' >
                     This should be the title
                 </a>
                 <span id='title_stars_2' name='title_stars_2'>  </span>
             </h3>
-            <p class='desc clk'>
+            <p class="search-item__body">
                 This should be the content.
             </p>
             <p>
@@ -56,7 +60,7 @@ class TestStartpageEngine(SearxTestCase):
                     Mis en surbrillance
                 </A>
             </p>
-        </div>
+        </li>
         """
         response = mock.Mock(text=html.encode('utf-8'))
         results = startpage.response(response)
@@ -67,14 +71,14 @@ class TestStartpageEngine(SearxTestCase):
         self.assertEqual(results[0]['content'], 'This should be the content.')
 
         html = """
-        <div class='result' style=' *width : auto; *margin-right : 10%;'>
+        <li class="search-result search-item">
             <h3>
                 <a href='http://www.google.com/aclk?sa=l&ai=C' id='title_2' name='title_2' >
                     This should be the title
                 </a>
                 <span id='title_stars_2' name='title_stars_2'>  </span>
             </h3>
-            <p class='desc clk'>
+            <p class="search-item__body">
                 This should be the content.
             </p>
             <p>
@@ -92,20 +96,20 @@ class TestStartpageEngine(SearxTestCase):
                     Mis en surbrillance
                 </A>
             </p>
-        </div>
-        <div class='result' style=' *width : auto; *margin-right : 10%;'>
+        </li>
+        <li class="search-result search-item">
             <h3>
                 <span id='title_stars_2' name='title_stars_2'>  </span>
             </h3>
-            <p class='desc clk'>
+            <p class="search-item__body">
                 This should be the content.
             </p>
             <p>
                 <span class='url'>www.speed<b>test</b>.net/fr/
                 </span>
             </p>
-        </div>
-        <div class='result' style=' *width : auto; *margin-right : 10%;'>
+        </li>
+        <li class="search-result search-item">
             <h3>
                 <a href='http://this.should.be.the.link/' id='title_2' name='title_2' >
                     This should be the title
@@ -127,7 +131,7 @@ class TestStartpageEngine(SearxTestCase):
                     Mis en surbrillance
                 </A>
             </p>
-        </div>
+        </li>
         """
         response = mock.Mock(text=html.encode('utf-8'))
         results = startpage.response(response)

tests/unit/engines/test_subtitleseeker.py

@@ -73,6 +73,42 @@ class TestSubtitleseekerEngine(SearxTestCase):
         results = subtitleseeker.response(response)
         self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/')
 
+        html = """
+        <div class="boxRows">
+            <div class="boxRowsInner" style="width:600px;">
+                <img src="http://static.subtitleseeker.com/images/movie.gif"
+                    style="width:16px; height:16px;" class="icon">
+                <a href="http://this.is.the.url/"
+                    class="blue" title="Title subtitle" >
+                    This is the Title
+                </a>
+            </div>
+            <div class="boxRowsInner f12b red" style="width:70px;">
+                1998
+            </div>
+            <div class="boxRowsInner grey-web f12" style="width:120px;">
+                <img src="http://static.subtitleseeker.com/images/basket_put.png"
+                    style="width:16px; height:16px;" class="icon">
+                1039 Subs
+            </div>
+            <div class="boxRowsInner grey-web f10" style="width:130px;">
+                <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
+                    style="width:16px; height:16px;" class="icon">
+                1 hours ago
+            </div>
+            <div class="clear"></div>
+        </div>
+        """
+        dicto['language'] = 'all'
+        response = mock.Mock(text=html, search_params=dicto)
+        results = subtitleseeker.response(response)
+        self.assertEqual(type(results), list)
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['title'], 'This is the Title')
+        self.assertEqual(results[0]['url'], 'http://this.is.the.url/')
+        self.assertIn('1998', results[0]['content'])
+        self.assertIn('1039 Subs', results[0]['content'])
+
         html = """
         <div class="boxRows">
             <div class="boxRowsInner" style="width:600px;">

tests/unit/engines/test_swisscows.py

@@ -8,6 +8,7 @@ class TestSwisscowsEngine(SearxTestCase):
 
     def test_request(self):
         swisscows.supported_languages = ['de-AT', 'de-DE']
+        swisscows.language_aliases = {}
         query = 'test_query'
         dicto = defaultdict(dict)
         dicto['pageno'] = 1
@@ -19,6 +20,11 @@ class TestSwisscowsEngine(SearxTestCase):
         self.assertTrue('uiLanguage=de' in params['url'])
         self.assertTrue('region=de-DE' in params['url'])
 
+        dicto['language'] = 'all'
+        params = swisscows.request(query, dicto)
+        self.assertTrue('uiLanguage=browser' in params['url'])
+        self.assertTrue('region=browser' in params['url'])
+
         dicto['category'] = 'images'
         params = swisscows.request(query, dicto)
         self.assertIn('image', params['url'])

tests/unit/engines/test_twitter.py

@@ -20,6 +20,12 @@ class TestTwitterEngine(SearxTestCase):
         self.assertIn('lang', params['cookies'])
         self.assertIn('fr', params['cookies']['lang'])
 
+        dicto['language'] = 'all'
+        params = twitter.request(query, dicto)
+        self.assertIn('cookies', params)
+        self.assertIn('lang', params['cookies'])
+        self.assertIn('en', params['cookies']['lang'])
+
     def test_response(self):
         self.assertRaises(AttributeError, twitter.response, None)
         self.assertRaises(AttributeError, twitter.response, [])

tests/unit/engines/test_wikidata.py

@@ -11,11 +11,13 @@ class TestWikidataEngine(SearxTestCase):
     def test_request(self):
         query = 'test_query'
         dicto = defaultdict(dict)
+        dicto['language'] = 'all'
         params = wikidata.request(query, dicto)
         self.assertIn('url', params)
         self.assertIn(query, params['url'])
         self.assertIn('wikidata.org', params['url'])
 
+        dicto['language'] = 'es_ES'
         params = wikidata.request(query, dicto)
         self.assertIn(query, params['url'])
 
@@ -27,6 +29,7 @@ class TestWikidataEngine(SearxTestCase):
         self.assertRaises(AttributeError, wikidata.response, '[]')
 
         wikidata.supported_languages = ['en', 'es']
+        wikidata.language_aliases = {}
         response = mock.Mock(text='<html></html>', search_params={"language": "en"})
         self.assertEqual(wikidata.response(response), [])
 

tests/unit/engines/test_wikipedia.py

@@ -8,7 +8,8 @@ from searx.testing import SearxTestCase
 class TestWikipediaEngine(SearxTestCase):
 
     def test_request(self):
-        wikipedia.supported_languages = ['fr', 'en']
+        wikipedia.supported_languages = ['fr', 'en', 'no']
+        wikipedia.language_aliases = {'nb': 'no'}
 
         query = 'test_query'
         dicto = defaultdict(dict)
@@ -25,10 +26,17 @@ class TestWikipediaEngine(SearxTestCase):
         self.assertIn('Test_Query', params['url'])
         self.assertNotIn('test_query', params['url'])
 
-        dicto['language'] = 'xx'
+        dicto['language'] = 'nb'
+        params = wikipedia.request(query, dicto)
+        self.assertIn('no.wikipedia.org', params['url'])
+        dicto['language'] = 'all'
         params = wikipedia.request(query, dicto)
         self.assertIn('en', params['url'])
 
+        dicto['language'] = 'xx'
+        params = wikipedia.request(query, dicto)
+        self.assertIn('en.wikipedia.org', params['url'])
+
     def test_response(self):
         dicto = defaultdict(dict)
         dicto['language'] = 'fr'

tests/unit/engines/test_yacy.py

@@ -17,6 +17,11 @@ class TestYacyEngine(SearxTestCase):
         self.assertIn('localhost', params['url'])
         self.assertIn('fr', params['url'])
 
+        dicto['language'] = 'all'
+        params = yacy.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertNotIn('lr=lang_', params['url'])
+
     def test_response(self):
         self.assertRaises(AttributeError, yacy.response, None)
         self.assertRaises(AttributeError, yacy.response, [])

tests/unit/engines/test_yahoo.py

@@ -50,6 +50,13 @@ class TestYahooEngine(SearxTestCase):
         self.assertIn('zh_cht', params['url'])
         self.assertIn('zh_cht', params['cookies']['sB'])
 
+        dicto['language'] = 'all'
+        params = yahoo.request(query, dicto)
+        self.assertIn('cookies', params)
+        self.assertIn('sB', params['cookies'])
+        self.assertIn('en', params['cookies']['sB'])
+        self.assertIn('en', params['url'])
+
     def test_no_url_in_request_year_time_range(self):
         dicto = defaultdict(dict)
         query = 'test_query'

tests/unit/engines/test_yahoo_news.py

@@ -23,6 +23,13 @@ class TestYahooNewsEngine(SearxTestCase):
         self.assertIn('sB', params['cookies'])
         self.assertIn('fr', params['cookies']['sB'])
 
+        dicto['language'] = 'all'
+        params = yahoo_news.request(query, dicto)
+        self.assertIn('cookies', params)
+        self.assertIn('sB', params['cookies'])
+        self.assertIn('en', params['cookies']['sB'])
+        self.assertIn('en', params['url'])
+
     def test_sanitize_url(self):
         url = "test.url"
         self.assertEqual(url, yahoo_news.sanitize_url(url))

tests/unit/test_preferences.py

@@ -90,30 +90,25 @@ class TestSettings(SearxTestCase):
 
     # search language settings
     def test_lang_setting_valid_choice(self):
-        setting = SearchLanguageSetting('en', choices=['de', 'en'])
+        setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])
         setting.parse('de')
         self.assertEquals(setting.get_value(), 'de')
 
     def test_lang_setting_invalid_choice(self):
-        setting = SearchLanguageSetting('en', choices=['de', 'en'])
+        setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])
         setting.parse('xx')
-        self.assertEquals(setting.get_value(), 'en')
+        self.assertEquals(setting.get_value(), 'all')
 
     def test_lang_setting_old_cookie_choice(self):
-        setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])
+        setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])
         setting.parse('es_XA')
         self.assertEquals(setting.get_value(), 'es')
 
     def test_lang_setting_old_cookie_format(self):
-        setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES'])
+        setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])
         setting.parse('es_ES')
         self.assertEquals(setting.get_value(), 'es-ES')
 
-    def test_lang_setting_old_default(self):
-        setting = SearchLanguageSetting('en', choices=['en', 'es', 'de'])
-        setting.parse('all')
-        self.assertEquals(setting.get_value(), 'en')
-
     # plugins settings
     def test_plugins_setting_all_default_enabled(self):
         plugin1 = PluginStub('plugin1', True)