jandieman
/
searxng
mirror of https://github.com/searxng/searxng.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

[fix] flickr engine

This commit is contained in:
Alexandre Flament 2015-05-27 13:49:25 +02:00
parent 37eb0fb130
commit 7afd02c94e
2 changed files with 74 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
searx/engines/flickr_noapi.py
View File

@ -25,7 +25,7 @@ categories = ['images']
url = 'https://www.flickr.com/' url = 'https://www.flickr.com/'
search_url = url + 'search?{query}&page={page}' search_url = url + 'search?{query}&page={page}'
photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
paging = True paging = True
@ -38,6 +38,7 @@ def build_flickr_url(user_id, photo_id):
def request(query, params): def request(query, params):
params['url'] = search_url.format(query=urlencode({'text': query}), params['url'] = search_url.format(query=urlencode({'text': query}),
page=params['pageno']) page=params['pageno'])
return params return params
@ -75,7 +76,7 @@ def response(resp):
logger.debug('cannot find valid image size: {0}'.format(repr(photo))) logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
continue continue
if 'id' not in photo['owner']: if 'ownerNsid' not in photo:
continue continue
# For a bigger thumbnail, keep only the url_z, not the url_n # For a bigger thumbnail, keep only the url_z, not the url_n
@ -86,20 +87,14 @@ def response(resp):
else: else:
thumbnail_src = img_src thumbnail_src = img_src
url = build_flickr_url(photo['owner']['id'], photo['id']) url = build_flickr_url(photo['ownerNsid'], photo['id'])
title = photo.get('title', '') title = photo.get('title', '')
content = '<span class="photo-author">' +\ content = '<span class="photo-author">' +\
photo['owner']['username'] +\ photo['username'] +\
'</span><br />' '</span><br />'
if 'description' in photo:
content = content +\
'<span class="description">' +\
photo['description'] +\
'</span>'
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,

248
searx/tests/engines/test_flickr_noapi.py
View File

@ -26,19 +26,29 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertRaises(AttributeError, flickr_noapi.response, '') self.assertRaises(AttributeError, flickr_noapi.response, '')
self.assertRaises(AttributeError, flickr_noapi.response, '[]') self.assertRaises(AttributeError, flickr_noapi.response, '[]')
response = mock.Mock(text='"search-photos-models","photos":{},"totalItems":') response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":')
self.assertEqual(flickr_noapi.response(response), []) self.assertEqual(flickr_noapi.response(response), [])
response = mock.Mock(text='search-photos-models","photos":{"data": []},"totalItems":') response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":')
self.assertEqual(flickr_noapi.response(response), []) self.assertEqual(flickr_noapi.response(response), [])
# everthing is ok test
json = """ json = """
"search-photos-models","photos": "search-photos-lite-models","photos":
{ {
"_data": [ "_data": [
{ {
"_flickrModelRegistry": "photo-models", "_flickrModelRegistry": "photo-lite-models",
"title": "This is the title", "title": "This is the title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": { "sizes": {
"c": { "c": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg", "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg",
@ -117,40 +127,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
"key": "z" "key": "z"
} }
}, }
"canComment": false,
"rotation": 0,
"owner": {
"_flickrModelRegistry": "person-models",
"pathAlias": "klink692",
"username": "Owner",
"buddyicon": {
"retina": null,
"large": null,
"medium": null,
"small": null,
"default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
},
"isPro": true,
"id": "59729010@N00"
},
"engagement": {
"_flickrModelRegistry": "photo-engagement-models",
"ownerNsid": "59729010@N00",
"faveCount": 21,
"commentCount": 14,
"viewCount": 10160,
"id": "14001294434"
},
"description": "Description",
"isHD": false,
"secret": "410f653777",
"canAddMeta": false,
"license": 0,
"oWidth": 1803,
"oHeight": 2669,
"safetyLevel": 0,
"id": "14001294434"
} }
], ],
"fetchedStart": true, "fetchedStart": true,
@ -168,15 +145,24 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertIn('k.jpg', results[0]['img_src']) self.assertIn('k.jpg', results[0]['img_src'])
self.assertIn('n.jpg', results[0]['thumbnail_src']) self.assertIn('n.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['content']) self.assertIn('Owner', results[0]['content'])
self.assertIn('Description', results[0]['content'])
# no n size, only the z size
json = """ json = """
"search-photos-models","photos": "search-photos-lite-models","photos":
{ {
"_data": [ "_data": [
{ {
"_flickrModelRegistry": "photo-models", "_flickrModelRegistry": "photo-lite-models",
"title": "This is the title", "title": "This is the title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": { "sizes": {
"z": { "z": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
@ -185,40 +171,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
"key": "z" "key": "z"
} }
}, }
"canComment": false,
"rotation": 0,
"owner": {
"_flickrModelRegistry": "person-models",
"pathAlias": "klink692",
"username": "Owner",
"buddyicon": {
"retina": null,
"large": null,
"medium": null,
"small": null,
"default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
},
"isPro": true,
"id": "59729010@N00"
},
"engagement": {
"_flickrModelRegistry": "photo-engagement-models",
"ownerNsid": "59729010@N00",
"faveCount": 21,
"commentCount": 14,
"viewCount": 10160,
"id": "14001294434"
},
"description": "Description",
"isHD": false,
"secret": "410f653777",
"canAddMeta": false,
"license": 0,
"oWidth": 1803,
"oHeight": 2669,
"safetyLevel": 0,
"id": "14001294434"
} }
], ],
"fetchedStart": true, "fetchedStart": true,
@ -235,15 +188,24 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertIn('z.jpg', results[0]['img_src']) self.assertIn('z.jpg', results[0]['img_src'])
self.assertIn('z.jpg', results[0]['thumbnail_src']) self.assertIn('z.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['content']) self.assertIn('Owner', results[0]['content'])
self.assertIn('Description', results[0]['content'])
# no z or n size
json = """ json = """
"search-photos-models","photos": "search-photos-lite-models","photos":
{ {
"_data": [ "_data": [
{ {
"_flickrModelRegistry": "photo-models", "_flickrModelRegistry": "photo-lite-models",
"title": "This is the title", "title": "This is the title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": { "sizes": {
"o": { "o": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
@ -252,39 +214,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
"key": "o" "key": "o"
} }
}, }
"canComment": false,
"rotation": 0,
"owner": {
"_flickrModelRegistry": "person-models",
"pathAlias": "klink692",
"username": "Owner",
"buddyicon": {
"retina": null,
"large": null,
"medium": null,
"small": null,
"default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
},
"isPro": true,
"id": "59729010@N00"
},
"engagement": {
"_flickrModelRegistry": "photo-engagement-models",
"ownerNsid": "59729010@N00",
"faveCount": 21,
"commentCount": 14,
"viewCount": 10160,
"id": "14001294434"
},
"isHD": false,
"secret": "410f653777",
"canAddMeta": false,
"license": 0,
"oWidth": 1803,
"oHeight": 2669,
"safetyLevel": 0,
"id": "14001294434"
} }
], ],
"fetchedStart": true, "fetchedStart": true,
@ -302,48 +232,25 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertIn('o.jpg', results[0]['thumbnail_src']) self.assertIn('o.jpg', results[0]['thumbnail_src'])
self.assertIn('Owner', results[0]['content']) self.assertIn('Owner', results[0]['content'])
# no image test
json = """ json = """
"search-photos-models","photos": "search-photos-lite-models","photos":
{ {
"_data": [ "_data": [
{ {
"_flickrModelRegistry": "photo-models", "_flickrModelRegistry": "photo-lite-models",
"title": "This is the title", "title": "This is the title",
"sizes": {
},
"canComment": false,
"rotation": 0,
"owner": {
"_flickrModelRegistry": "person-models",
"pathAlias": "klink692",
"username": "Owner", "username": "Owner",
"buddyicon": { "pathAlias": "klink692",
"retina": null, "realname": "Owner",
"large": null,
"medium": null,
"small": null,
"default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
},
"isPro": true,
"id": "59729010@N00"
},
"engagement": {
"_flickrModelRegistry": "photo-engagement-models",
"ownerNsid": "59729010@N00",
"faveCount": 21,
"commentCount": 14,
"viewCount": 10160,
"id": "14001294434"
},
"description": "Description",
"isHD": false,
"secret": "410f653777",
"canAddMeta": false,
"license": 0, "license": 0,
"oWidth": 1803, "ownerNsid": "59729010@N00",
"oHeight": 2669, "canComment": false,
"safetyLevel": 0, "commentCount": 14,
"id": "14001294434" "faveCount": 21,
"id": "14001294434",
"sizes": {
}
} }
], ],
"fetchedStart": true, "fetchedStart": true,
@ -356,6 +263,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
self.assertEqual(len(results), 0) self.assertEqual(len(results), 0)
# null test
json = """ json = """
"search-photos-models","photos": "search-photos-models","photos":
{ {
@ -370,13 +278,22 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
self.assertEqual(len(results), 0) self.assertEqual(len(results), 0)
# no ownerNsid test
json = """ json = """
"search-photos-models","photos": "search-photos-lite-models","photos":
{ {
"_data": [ "_data": [
{ {
"_flickrModelRegistry": "photo-models", "_flickrModelRegistry": "photo-lite-models",
"title": "This is the title", "title": "This is the title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": { "sizes": {
"o": { "o": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
@ -385,39 +302,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
"key": "o" "key": "o"
} }
}, }
"canComment": false,
"rotation": 0,
"owner": {
"_flickrModelRegistry": "person-models",
"pathAlias": "klink692",
"username": "Owner",
"buddyicon": {
"retina": null,
"large": null,
"medium": null,
"small": null,
"default": "//c1.staticflickr.com/9/8108/buddyicons/59729010@N00.jpg?1361642376#59729010@N00"
},
"isPro": true
},
"engagement": {
"_flickrModelRegistry": "photo-engagement-models",
"ownerNsid": "59729010@N00",
"faveCount": 21,
"commentCount": 14,
"viewCount": 10160,
"id": "14001294434"
},
"description": "Description",
"isHD": false,
"secret": "410f653777",
"canAddMeta": false,
"license": 0,
"oWidth": 1803,
"oHeight": 2669,
"safetyLevel": 0,
"id": "14001294434"
} }
], ],
"fetchedStart": true, "fetchedStart": true,
@ -430,6 +315,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
self.assertEqual(len(results), 0) self.assertEqual(len(results), 0)
# garbage test
json = """ json = """
{"toto":[ {"toto":[
{"id":200,"name":"Artist Name", {"id":200,"name":"Artist Name",