mirror of https://github.com/searxng/searxng.git
[enh] flickr_noapi: use complete JSON data block, add 'content', 'img_format', 'source', etc. (#1571)
Fetch complete JSON data block, use legend to extract images. Unquote urlencoded strings. Add image description as 'content'. Add 'img_format' and 'source' data (needs PR #1567 to enable this data to be displayed). Show images which lack ownerid instead of discarding them.
This commit is contained in:
Frank de Lange
Alexandre Flament
parent
c6ac39bcea
commit
cbc5e13275
|
|
@ -16,8 +16,7 @@ from json import loads
|
|||
from time import time
|
||||
import re
|
||||
from searx.engines import logger
|
||||
from searx.url_utils import urlencode
|
||||
|
||||
from searx.url_utils import urlencode, unquote
|
||||
|
||||
logger = logger.getChild('flickr-noapi')
|
||||
|
||||
|
|
@ -27,7 +26,7 @@ url = 'https://www.flickr.com/'
|
|||
search_url = url + 'search?{query}&page={page}'
|
||||
time_range_url = '&min_upload_date={start}&max_upload_date={end}'
|
||||
photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
|
||||
regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
|
||||
modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M)
|
||||
image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
|
||||
|
||||
paging = True
|
||||
|
|
@ -57,40 +56,45 @@ def request(query, params):
|
|||
def response(resp):
|
||||
results = []
|
||||
|
||||
matches = regex.search(resp.text)
|
||||
matches = modelexport_re.search(resp.text)
|
||||
|
||||
if matches is None:
|
||||
return results
|
||||
|
||||
match = matches.group(1)
|
||||
search_results = loads(match)
|
||||
model_export = loads(match)
|
||||
|
||||
if '_data' not in search_results:
|
||||
return []
|
||||
if 'legend' not in model_export:
|
||||
return results
|
||||
|
||||
photos = search_results['_data']
|
||||
legend = model_export['legend']
|
||||
|
||||
for photo in photos:
|
||||
# handle empty page
|
||||
if not legend or not legend[0]:
|
||||
return results
|
||||
|
||||
# In paged configuration, the first pages' photos
|
||||
# are represented by a None object
|
||||
if photo is None:
|
||||
continue
|
||||
for index in legend:
|
||||
photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][int(index[4])]
|
||||
author = unquote(photo.get('realname', ''))
|
||||
source = unquote(photo.get('username', '')) + ' @ Flickr'
|
||||
title = unquote(photo.get('title', ''))
|
||||
content = unquote(photo.get('description', ''))
|
||||
|
||||
img_src = None
|
||||
# From the biggest to the lowest format
|
||||
for image_size in image_sizes:
|
||||
if image_size in photo['sizes']:
|
||||
img_src = photo['sizes'][image_size]['url']
|
||||
img_format = 'jpg ' \
|
||||
+ str(photo['sizes'][image_size]['width']) \
|
||||
+ 'x' \
|
||||
+ str(photo['sizes'][image_size]['height'])
|
||||
break
|
||||
|
||||
if not img_src:
|
||||
logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
|
||||
continue
|
||||
|
||||
if 'ownerNsid' not in photo:
|
||||
continue
|
||||
|
||||
# For a bigger thumbnail, keep only the url_z, not the url_n
|
||||
if 'n' in photo['sizes']:
|
||||
thumbnail_src = photo['sizes']['n']['url']
|
||||
|
|
@ -99,19 +103,20 @@ def response(resp):
|
|||
else:
|
||||
thumbnail_src = img_src
|
||||
|
||||
if 'ownerNsid' not in photo:
|
||||
# should not happen, disowned photo? Show it anyway
|
||||
url = img_src
|
||||
else:
|
||||
url = build_flickr_url(photo['ownerNsid'], photo['id'])
|
||||
|
||||
title = photo.get('title', '')
|
||||
|
||||
author = photo['username']
|
||||
|
||||
# append result
|
||||
results.append({'url': url,
|
||||
'title': title,
|
||||
'img_src': img_src,
|
||||
'thumbnail_src': thumbnail_src,
|
||||
'content': '',
|
||||
'content': content,
|
||||
'author': author,
|
||||
'source': source,
|
||||
'img_format': img_format,
|
||||
'template': 'images.html'})
|
||||
|
||||
return results
|
||||
|
|
|
|||
|
|
@ -27,20 +27,33 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
self.assertRaises(AttributeError, flickr_noapi.response, '')
|
||||
self.assertRaises(AttributeError, flickr_noapi.response, '[]')
|
||||
|
||||
response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":')
|
||||
response = mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{}}]}}')
|
||||
self.assertEqual(flickr_noapi.response(response), [])
|
||||
|
||||
response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":')
|
||||
response = \
|
||||
mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{"_data":[]}}]}}')
|
||||
self.assertEqual(flickr_noapi.response(response), [])
|
||||
|
||||
# everthing is ok test
|
||||
json = """
|
||||
"search-photos-lite-models","photos":
|
||||
modelExport: {
|
||||
"legend": [
|
||||
[
|
||||
"search-photos-lite-models",
|
||||
"0",
|
||||
"photos",
|
||||
"_data",
|
||||
"0"
|
||||
]
|
||||
],
|
||||
"main": {
|
||||
"search-photos-lite-models": [
|
||||
{
|
||||
"photos": {
|
||||
"_data": [
|
||||
{
|
||||
"_flickrModelRegistry": "photo-lite-models",
|
||||
"title": "This is the title",
|
||||
"title": "This%20is%20the%20title",
|
||||
"username": "Owner",
|
||||
"pathAlias": "klink692",
|
||||
"realname": "Owner",
|
||||
|
|
@ -130,13 +143,16 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"fetchedStart": true,
|
||||
"fetchedEnd": false,
|
||||
"totalItems": "4386039"
|
||||
},"totalItems":
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"""
|
||||
json = json.replace('\r\n', '').replace('\n', '').replace('\r', '')
|
||||
# Flickr serves search results in a json block named 'modelExport' buried inside a script tag,
|
||||
# this json is served as a single line terminating with a comma.
|
||||
json = ''.join(json.split()) + ',\n'
|
||||
response = mock.Mock(text=json)
|
||||
results = flickr_noapi.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
|
|
@ -149,12 +165,24 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
|
||||
# no n size, only the z size
|
||||
json = """
|
||||
"search-photos-lite-models","photos":
|
||||
modelExport: {
|
||||
"legend": [
|
||||
[
|
||||
"search-photos-lite-models",
|
||||
"0",
|
||||
"photos",
|
||||
"_data",
|
||||
"0"
|
||||
]
|
||||
],
|
||||
"main": {
|
||||
"search-photos-lite-models": [
|
||||
{
|
||||
"photos": {
|
||||
"_data": [
|
||||
{
|
||||
"_flickrModelRegistry": "photo-lite-models",
|
||||
"title": "This is the title",
|
||||
"title": "This%20is%20the%20title",
|
||||
"username": "Owner",
|
||||
"pathAlias": "klink692",
|
||||
"realname": "Owner",
|
||||
|
|
@ -174,12 +202,14 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"fetchedStart": true,
|
||||
"fetchedEnd": false,
|
||||
"totalItems": "4386039"
|
||||
},"totalItems":
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"""
|
||||
json = ''.join(json.split()) + ',\n'
|
||||
response = mock.Mock(text=json)
|
||||
results = flickr_noapi.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
|
|
@ -192,12 +222,24 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
|
||||
# no z or n size
|
||||
json = """
|
||||
"search-photos-lite-models","photos":
|
||||
modelExport: {
|
||||
"legend": [
|
||||
[
|
||||
"search-photos-lite-models",
|
||||
"0",
|
||||
"photos",
|
||||
"_data",
|
||||
"0"
|
||||
]
|
||||
],
|
||||
"main": {
|
||||
"search-photos-lite-models": [
|
||||
{
|
||||
"photos": {
|
||||
"_data": [
|
||||
{
|
||||
"_flickrModelRegistry": "photo-lite-models",
|
||||
"title": "This is the title",
|
||||
"title": "This%20is%20the%20title",
|
||||
"username": "Owner",
|
||||
"pathAlias": "klink692",
|
||||
"realname": "Owner",
|
||||
|
|
@ -217,12 +259,14 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"fetchedStart": true,
|
||||
"fetchedEnd": false,
|
||||
"totalItems": "4386039"
|
||||
},"totalItems":
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"""
|
||||
json = ''.join(json.split()) + ',\n'
|
||||
response = mock.Mock(text=json)
|
||||
results = flickr_noapi.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
|
|
@ -235,8 +279,20 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
|
||||
# no image test
|
||||
json = """
|
||||
"search-photos-lite-models","photos":
|
||||
modelExport: {
|
||||
"legend": [
|
||||
[
|
||||
"search-photos-lite-models",
|
||||
"0",
|
||||
"photos",
|
||||
"_data",
|
||||
"0"
|
||||
]
|
||||
],
|
||||
"main": {
|
||||
"search-photos-lite-models": [
|
||||
{
|
||||
"photos": {
|
||||
"_data": [
|
||||
{
|
||||
"_flickrModelRegistry": "photo-lite-models",
|
||||
|
|
@ -253,12 +309,14 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
"sizes": {
|
||||
}
|
||||
}
|
||||
],
|
||||
"fetchedStart": true,
|
||||
"fetchedEnd": false,
|
||||
"totalItems": "4386039"
|
||||
},"totalItems":
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"""
|
||||
json = ''.join(json.split()) + ',\n'
|
||||
response = mock.Mock(text=json)
|
||||
results = flickr_noapi.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
|
|
@ -266,51 +324,20 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
|
||||
# null test
|
||||
json = """
|
||||
"search-photos-models","photos":
|
||||
modelExport: {
|
||||
"legend": [null],
|
||||
"main": {
|
||||
"search-photos-lite-models": [
|
||||
{
|
||||
"_data": [null],
|
||||
"fetchedStart": true,
|
||||
"fetchedEnd": false,
|
||||
"totalItems": "4386039"
|
||||
},"totalItems":
|
||||
"""
|
||||
response = mock.Mock(text=json)
|
||||
results = flickr_noapi.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
self.assertEqual(len(results), 0)
|
||||
|
||||
# no ownerNsid test
|
||||
json = """
|
||||
"search-photos-lite-models","photos":
|
||||
{
|
||||
"_data": [
|
||||
{
|
||||
"_flickrModelRegistry": "photo-lite-models",
|
||||
"title": "This is the title",
|
||||
"username": "Owner",
|
||||
"pathAlias": "klink692",
|
||||
"realname": "Owner",
|
||||
"license": 0,
|
||||
"canComment": false,
|
||||
"commentCount": 14,
|
||||
"faveCount": 21,
|
||||
"id": "14001294434",
|
||||
"sizes": {
|
||||
"o": {
|
||||
"displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
|
||||
"width": 433,
|
||||
"height": 640,
|
||||
"url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
|
||||
"key": "o"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"fetchedStart": true,
|
||||
"fetchedEnd": false,
|
||||
"totalItems": "4386039"
|
||||
},"totalItems":
|
||||
"photos": {
|
||||
"_data": [null]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
"""
|
||||
json = ''.join(json.split()) + ',\n'
|
||||
response = mock.Mock(text=json)
|
||||
results = flickr_noapi.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
|
|
@ -323,6 +350,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
|
|||
"link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}
|
||||
]}
|
||||
"""
|
||||
json = ''.join(json.split()) + ',\n'
|
||||
response = mock.Mock(text=json)
|
||||
results = flickr_noapi.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
|
|
|
|||
Loading…
Reference in New Issue