2015-02-08 13:12:14 +00:00
|
|
|
from collections import defaultdict
|
|
|
|
|
import mock
|
|
|
|
|
from searx.engines import gigablast
|
|
|
|
|
from searx.testing import SearxTestCase
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestGigablastEngine(SearxTestCase):
|
|
|
|
|
|
|
|
|
|
def test_request(self):
|
|
|
|
|
query = 'test_query'
|
|
|
|
|
dicto = defaultdict(dict)
|
|
|
|
|
dicto['pageno'] = 0
|
2016-11-30 17:43:03 +00:00
|
|
|
dicto['safesearch'] = 0
|
2015-12-23 17:43:35 +00:00
|
|
|
dicto['language'] = 'all'
|
2015-02-08 13:12:14 +00:00
|
|
|
params = gigablast.request(query, dicto)
|
|
|
|
|
self.assertTrue('url' in params)
|
|
|
|
|
self.assertTrue(query in params['url'])
|
|
|
|
|
self.assertTrue('gigablast.com' in params['url'])
|
2016-12-14 05:51:15 +00:00
|
|
|
self.assertTrue('xx' in params['url'])
|
|
|
|
|
|
|
|
|
|
dicto['language'] = 'en-US'
|
|
|
|
|
params = gigablast.request(query, dicto)
|
|
|
|
|
self.assertTrue('en' in params['url'])
|
|
|
|
|
self.assertFalse('en-US' in params['url'])
|
2015-02-08 13:12:14 +00:00
|
|
|
|
|
|
|
|
def test_response(self):
|
|
|
|
|
self.assertRaises(AttributeError, gigablast.response, None)
|
|
|
|
|
self.assertRaises(AttributeError, gigablast.response, [])
|
|
|
|
|
self.assertRaises(AttributeError, gigablast.response, '')
|
|
|
|
|
self.assertRaises(AttributeError, gigablast.response, '[]')
|
|
|
|
|
|
2016-02-01 20:41:09 +00:00
|
|
|
response = mock.Mock(text='{"results": []}')
|
2015-02-08 13:12:14 +00:00
|
|
|
self.assertEqual(gigablast.response(response), [])
|
|
|
|
|
|
2016-02-01 20:41:09 +00:00
|
|
|
json = """{"results": [
|
|
|
|
|
{
|
2016-02-01 20:53:03 +00:00
|
|
|
"title":"South by Southwest 2016",
|
2016-02-01 20:41:09 +00:00
|
|
|
"dmozEntry":{
|
|
|
|
|
"dmozCatId":1041152,
|
|
|
|
|
"directCatId":1,
|
2016-02-01 20:53:03 +00:00
|
|
|
"dmozCatStr":"Top: Regional: North America: United States",
|
2016-02-01 20:41:09 +00:00
|
|
|
"dmozTitle":"South by Southwest (SXSW)",
|
2016-02-01 20:53:03 +00:00
|
|
|
"dmozSum":"Annual music, film, and interactive conference.",
|
2016-02-01 20:41:09 +00:00
|
|
|
"dmozAnchor":""
|
|
|
|
|
},
|
|
|
|
|
"dmozEntry":{
|
|
|
|
|
"dmozCatId":763945,
|
|
|
|
|
"directCatId":1,
|
2016-02-01 20:53:03 +00:00
|
|
|
"dmozCatStr":"Top: Regional: North America: United States",
|
2016-02-01 20:41:09 +00:00
|
|
|
"dmozTitle":"South by Southwest (SXSW)",
|
|
|
|
|
"dmozSum":"",
|
|
|
|
|
"dmozAnchor":"www.sxsw.com"
|
|
|
|
|
},
|
|
|
|
|
"dmozEntry":{
|
|
|
|
|
"dmozCatId":761446,
|
|
|
|
|
"directCatId":1,
|
2016-02-01 20:53:03 +00:00
|
|
|
"dmozCatStr":"Top: Regional: North America: United States",
|
2016-02-01 20:41:09 +00:00
|
|
|
"dmozTitle":"South by Southwest (SXSW)",
|
2016-02-01 20:53:03 +00:00
|
|
|
"dmozSum":"Music, film, and interactive conference and festival.",
|
2016-02-01 20:41:09 +00:00
|
|
|
"dmozAnchor":""
|
|
|
|
|
},
|
|
|
|
|
"indirectDmozCatId":1041152,
|
|
|
|
|
"indirectDmozCatId":763945,
|
|
|
|
|
"indirectDmozCatId":761446,
|
|
|
|
|
"contentType":"html",
|
|
|
|
|
"sum":"This should be the content.",
|
|
|
|
|
"url":"www.sxsw.com",
|
|
|
|
|
"hopCount":0,
|
|
|
|
|
"size":" 102k",
|
|
|
|
|
"sizeInBytes":104306,
|
|
|
|
|
"bytesUsedToComputeSummary":70000,
|
|
|
|
|
"docId":269411794364,
|
|
|
|
|
"docScore":586571136.000000,
|
|
|
|
|
"summaryGenTimeMS":12,
|
|
|
|
|
"summaryTagdbLookupTimeMS":0,
|
|
|
|
|
"summaryTitleRecLoadTimeMS":1,
|
|
|
|
|
"site":"www.sxsw.com",
|
|
|
|
|
"spidered":1452203608,
|
|
|
|
|
"firstIndexedDateUTC":1444167123,
|
|
|
|
|
"contentHash32":2170650347,
|
|
|
|
|
"language":"English",
|
|
|
|
|
"langAbbr":"en"
|
|
|
|
|
}
|
|
|
|
|
]}
|
2015-02-08 13:12:14 +00:00
|
|
|
"""
|
2016-02-01 20:41:09 +00:00
|
|
|
response = mock.Mock(text=json)
|
2015-02-08 13:12:14 +00:00
|
|
|
results = gigablast.response(response)
|
|
|
|
|
self.assertEqual(type(results), list)
|
|
|
|
|
self.assertEqual(len(results), 1)
|
2016-02-01 20:53:03 +00:00
|
|
|
self.assertEqual(results[0]['title'], 'South by Southwest 2016')
|
2016-02-01 20:41:09 +00:00
|
|
|
self.assertEqual(results[0]['url'], 'www.sxsw.com')
|
2015-02-08 13:12:14 +00:00
|
|
|
self.assertEqual(results[0]['content'], 'This should be the content.')
|
2016-12-15 06:34:43 +00:00
|
|
|
|
|
|
|
|
def test_fetch_supported_languages(self):
|
|
|
|
|
html = """<html></html>"""
|
|
|
|
|
response = mock.Mock(text=html)
|
|
|
|
|
results = gigablast._fetch_supported_languages(response)
|
|
|
|
|
self.assertEqual(type(results), list)
|
|
|
|
|
self.assertEqual(len(results), 0)
|
|
|
|
|
|
|
|
|
|
html = """
|
|
|
|
|
<html>
|
|
|
|
|
<body>
|
|
|
|
|
<span id="menu2">
|
|
|
|
|
<a href="/search?&rxikd=1&qlang=xx"></a>
|
|
|
|
|
<a href="/search?&rxikd=1&qlang=en"></a>
|
2016-12-29 05:24:56 +00:00
|
|
|
<a href="/search?&rxikd=1&prepend=gblang%3Aen"></a>
|
|
|
|
|
<a href="/search?&rxikd=1&qlang=zh_"></a>
|
|
|
|
|
<a href="/search?&rxikd=1&prepend=gblang%3Azh_tw"></a>
|
2016-12-15 06:34:43 +00:00
|
|
|
</span>
|
|
|
|
|
</body>
|
|
|
|
|
</html>
|
|
|
|
|
"""
|
|
|
|
|
response = mock.Mock(text=html)
|
|
|
|
|
languages = gigablast._fetch_supported_languages(response)
|
|
|
|
|
self.assertEqual(type(languages), list)
|
|
|
|
|
self.assertEqual(len(languages), 2)
|
|
|
|
|
self.assertIn('en', languages)
|
2016-12-29 05:24:56 +00:00
|
|
|
self.assertIn('zh-TW', languages)
|
