jandieman
/
searxng
mirror of https://github.com/searxng/searxng.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

[enh] flickr_noapi: use complete JSON data block, add 'content', 'img_format', 'source', etc. (#1571) 

Fetch complete JSON data block, use legend to extract images. 
Unquote urlencoded strings.
Add image description as 'content'. 
Add 'img_format' and 'source' data (needs PR #1567 to enable this data to be displayed). 
Show images which lack ownerid instead of discarding them.
This commit is contained in:
Frank de Lange 2019-07-28 10:42:00 +02:00 committed by Alexandre Flament
parent c6ac39bcea
commit cbc5e13275
2 changed files with 277 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
searx/engines/flickr_noapi.py
View File

@ -16,8 +16,7 @@ from json import loads
from time import time from time import time
import re import re
from searx.engines import logger from searx.engines import logger
from searx.url_utils import urlencode from searx.url_utils import urlencode, unquote
logger = logger.getChild('flickr-noapi') logger = logger.getChild('flickr-noapi')
@ -27,7 +26,7 @@ url = 'https://www.flickr.com/'
search_url = url + 'search?{query}&page={page}' search_url = url + 'search?{query}&page={page}'
time_range_url = '&min_upload_date={start}&max_upload_date={end}' time_range_url = '&min_upload_date={start}&max_upload_date={end}'
photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M)
image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
paging = True paging = True
@ -57,40 +56,45 @@ def request(query, params):
def response(resp): def response(resp):
results = [] results = []
matches = regex.search(resp.text) matches = modelexport_re.search(resp.text)
if matches is None: if matches is None:
return results return results
match = matches.group(1) match = matches.group(1)
search_results = loads(match) model_export = loads(match)
if '_data' not in search_results: if 'legend' not in model_export:
return [] return results
photos = search_results['_data'] legend = model_export['legend']
for photo in photos: # handle empty page
if not legend or not legend[0]:
return results
# In paged configuration, the first pages' photos for index in legend:
# are represented by a None object photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][int(index[4])]
if photo is None: author = unquote(photo.get('realname', ''))
continue source = unquote(photo.get('username', '')) + ' @ Flickr'
title = unquote(photo.get('title', ''))
content = unquote(photo.get('description', ''))
img_src = None img_src = None
# From the biggest to the lowest format # From the biggest to the lowest format
for image_size in image_sizes: for image_size in image_sizes:
if image_size in photo['sizes']: if image_size in photo['sizes']:
img_src = photo['sizes'][image_size]['url'] img_src = photo['sizes'][image_size]['url']
img_format = 'jpg ' \
+ str(photo['sizes'][image_size]['width']) \
+ 'x' \
+ str(photo['sizes'][image_size]['height'])
break break
if not img_src: if not img_src:
logger.debug('cannot find valid image size: {0}'.format(repr(photo))) logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
continue continue
if 'ownerNsid' not in photo:
continue
# For a bigger thumbnail, keep only the url_z, not the url_n # For a bigger thumbnail, keep only the url_z, not the url_n
if 'n' in photo['sizes']: if 'n' in photo['sizes']:
thumbnail_src = photo['sizes']['n']['url'] thumbnail_src = photo['sizes']['n']['url']
@ -99,19 +103,20 @@ def response(resp):
else: else:
thumbnail_src = img_src thumbnail_src = img_src
url = build_flickr_url(photo['ownerNsid'], photo['id']) if 'ownerNsid' not in photo:
# should not happen, disowned photo? Show it anyway
url = img_src
else:
url = build_flickr_url(photo['ownerNsid'], photo['id'])
title = photo.get('title', '')
author = photo['username']
# append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,
'img_src': img_src, 'img_src': img_src,
'thumbnail_src': thumbnail_src, 'thumbnail_src': thumbnail_src,
'content': '', 'content': content,
'author': author, 'author': author,
'source': source,
'img_format': img_format,
'template': 'images.html'}) 'template': 'images.html'})
return results return results

470
tests/unit/engines/test_flickr_noapi.py
View File

@ -27,116 +27,132 @@ class TestFlickrNoapiEngine(SearxTestCase):
self.assertRaises(AttributeError, flickr_noapi.response, '') self.assertRaises(AttributeError, flickr_noapi.response, '')
self.assertRaises(AttributeError, flickr_noapi.response, '[]') self.assertRaises(AttributeError, flickr_noapi.response, '[]')
response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":') response = mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{}}]}}')
self.assertEqual(flickr_noapi.response(response), []) self.assertEqual(flickr_noapi.response(response), [])
response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":') response = \
mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{"_data":[]}}]}}')
self.assertEqual(flickr_noapi.response(response), []) self.assertEqual(flickr_noapi.response(response), [])
# everthing is ok test # everthing is ok test
json = """ json = """
"search-photos-lite-models","photos": modelExport: {
{ "legend": [
"_data": [ [
{ "search-photos-lite-models",
"_flickrModelRegistry": "photo-lite-models", "0",
"title": "This is the title", "photos",
"username": "Owner", "_data",
"pathAlias": "klink692", "0"
"realname": "Owner", ]
"license": 0, ],
"ownerNsid": "59729010@N00", "main": {
"canComment": false, "search-photos-lite-models": [
"commentCount": 14, {
"faveCount": 21, "photos": {
"id": "14001294434", "_data": [
"sizes": { {
"c": { "_flickrModelRegistry": "photo-lite-models",
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg", "title": "This%20is%20the%20title",
"width": 541, "username": "Owner",
"height": 800, "pathAlias": "klink692",
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg", "realname": "Owner",
"key": "c" "license": 0,
}, "ownerNsid": "59729010@N00",
"h": { "canComment": false,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg", "commentCount": 14,
"width": 1081, "faveCount": 21,
"height": 1600, "id": "14001294434",
"url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg", "sizes": {
"key": "h" "c": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg",
"k": { "width": 541,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg", "height": 800,
"width": 1383, "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg",
"height": 2048, "key": "c"
"url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg", },
"key": "k" "h": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg",
"l": { "width": 1081,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg", "height": 1600,
"width": 692, "url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg",
"height": 1024, "key": "h"
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg", },
"key": "l" "k": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg",
"m": { "width": 1383,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg", "height": 2048,
"width": 338, "url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg",
"height": 500, "key": "k"
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg", },
"key": "m" "l": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg",
"n": { "width": 692,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg", "height": 1024,
"width": 216, "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg",
"height": 320, "key": "l"
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg", },
"key": "n" "m": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg",
"q": { "width": 338,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg", "height": 500,
"width": 150, "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg",
"height": 150, "key": "m"
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg", },
"key": "q" "n": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg",
"s": { "width": 216,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg", "height": 320,
"width": 162, "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg",
"height": 240, "key": "n"
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg", },
"key": "s" "q": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg",
"sq": { "width": 150,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg", "height": 150,
"width": 75, "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg",
"height": 75, "key": "q"
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg", },
"key": "sq" "s": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg",
"t": { "width": 162,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg", "height": 240,
"width": 68, "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg",
"height": 100, "key": "s"
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg", },
"key": "t" "sq": {
}, "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg",
"z": { "width": 75,
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", "height": 75,
"width": 433, "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg",
"height": 640, "key": "sq"
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", },
"key": "z" "t": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg",
"width": 68,
"height": 100,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg",
"key": "t"
},
"z": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
"width": 433,
"height": 640,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
"key": "z"
}
}
}
]
} }
} }
} ]
], }
"fetchedStart": true, }
"fetchedEnd": false,
"totalItems": "4386039"
},"totalItems":
""" """
json = json.replace('\r\n', '').replace('\n', '').replace('\r', '') # Flickr serves search results in a json block named 'modelExport' buried inside a script tag,
# this json is served as a single line terminating with a comma.
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json) response = mock.Mock(text=json)
results = flickr_noapi.response(response) results = flickr_noapi.response(response)
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
@ -149,37 +165,51 @@ class TestFlickrNoapiEngine(SearxTestCase):
# no n size, only the z size # no n size, only the z size
json = """ json = """
"search-photos-lite-models","photos": modelExport: {
{ "legend": [
"_data": [ [
{ "search-photos-lite-models",
"_flickrModelRegistry": "photo-lite-models", "0",
"title": "This is the title", "photos",
"username": "Owner", "_data",
"pathAlias": "klink692", "0"
"realname": "Owner", ]
"license": 0, ],
"ownerNsid": "59729010@N00", "main": {
"canComment": false, "search-photos-lite-models": [
"commentCount": 14, {
"faveCount": 21, "photos": {
"id": "14001294434", "_data": [
"sizes": { {
"z": { "_flickrModelRegistry": "photo-lite-models",
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", "title": "This%20is%20the%20title",
"width": 433, "username": "Owner",
"height": 640, "pathAlias": "klink692",
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", "realname": "Owner",
"key": "z" "license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
"z": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
"width": 433,
"height": 640,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
"key": "z"
}
}
}
]
} }
} }
} ]
], }
"fetchedStart": true, }
"fetchedEnd": false,
"totalItems": "4386039"
},"totalItems":
""" """
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json) response = mock.Mock(text=json)
results = flickr_noapi.response(response) results = flickr_noapi.response(response)
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
@ -192,37 +222,51 @@ class TestFlickrNoapiEngine(SearxTestCase):
# no z or n size # no z or n size
json = """ json = """
"search-photos-lite-models","photos": modelExport: {
{ "legend": [
"_data": [ [
{ "search-photos-lite-models",
"_flickrModelRegistry": "photo-lite-models", "0",
"title": "This is the title", "photos",
"username": "Owner", "_data",
"pathAlias": "klink692", "0"
"realname": "Owner", ]
"license": 0, ],
"ownerNsid": "59729010@N00", "main": {
"canComment": false, "search-photos-lite-models": [
"commentCount": 14, {
"faveCount": 21, "photos": {
"id": "14001294434", "_data": [
"sizes": { {
"o": { "_flickrModelRegistry": "photo-lite-models",
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", "title": "This%20is%20the%20title",
"width": 433, "username": "Owner",
"height": 640, "pathAlias": "klink692",
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", "realname": "Owner",
"key": "o" "license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
"o": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
"width": 433,
"height": 640,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
"key": "o"
}
}
}
]
} }
} }
} ]
], }
"fetchedStart": true, }
"fetchedEnd": false,
"totalItems": "4386039"
},"totalItems":
""" """
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json) response = mock.Mock(text=json)
results = flickr_noapi.response(response) results = flickr_noapi.response(response)
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
@ -235,30 +279,44 @@ class TestFlickrNoapiEngine(SearxTestCase):
# no image test # no image test
json = """ json = """
"search-photos-lite-models","photos": modelExport: {
{ "legend": [
"_data": [ [
{ "search-photos-lite-models",
"_flickrModelRegistry": "photo-lite-models", "0",
"title": "This is the title", "photos",
"username": "Owner", "_data",
"pathAlias": "klink692", "0"
"realname": "Owner", ]
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
}
}
], ],
"fetchedStart": true, "main": {
"fetchedEnd": false, "search-photos-lite-models": [
"totalItems": "4386039" {
},"totalItems": "photos": {
"_data": [
{
"_flickrModelRegistry": "photo-lite-models",
"title": "This is the title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"ownerNsid": "59729010@N00",
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
}
}
]
}
}
]
}
}
""" """
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json) response = mock.Mock(text=json)
results = flickr_noapi.response(response) results = flickr_noapi.response(response)
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
@ -266,51 +324,20 @@ class TestFlickrNoapiEngine(SearxTestCase):
# null test # null test
json = """ json = """
"search-photos-models","photos": modelExport: {
{ "legend": [null],
"_data": [null], "main": {
"fetchedStart": true, "search-photos-lite-models": [
"fetchedEnd": false, {
"totalItems": "4386039" "photos": {
},"totalItems": "_data": [null]
"""
response = mock.Mock(text=json)
results = flickr_noapi.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
# no ownerNsid test
json = """
"search-photos-lite-models","photos":
{
"_data": [
{
"_flickrModelRegistry": "photo-lite-models",
"title": "This is the title",
"username": "Owner",
"pathAlias": "klink692",
"realname": "Owner",
"license": 0,
"canComment": false,
"commentCount": 14,
"faveCount": 21,
"id": "14001294434",
"sizes": {
"o": {
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
"width": 433,
"height": 640,
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
"key": "o"
} }
} }
} ]
], }
"fetchedStart": true, }
"fetchedEnd": false,
"totalItems": "4386039"
},"totalItems":
""" """
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json) response = mock.Mock(text=json)
results = flickr_noapi.response(response) results = flickr_noapi.response(response)
self.assertEqual(type(results), list) self.assertEqual(type(results), list)
@ -323,6 +350,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
"link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"} "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}
]} ]}
""" """
json = ''.join(json.split()) + ',\n'
response = mock.Mock(text=json) response = mock.Mock(text=json)
results = flickr_noapi.response(response) results = flickr_noapi.response(response)
self.assertEqual(type(results), list) self.assertEqual(type(results), list)