From cfdcbdd9b536ce859d6f4c1690d4b788c1d89463 Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Mon, 15 Dec 2014 03:21:25 +0100 Subject: [PATCH 1/3] Rework Flickr Engine Everything was redone to use the API. It needs an API key, but it's worth it. Everything works. Title, Image, Content, URL The API allow lots of things. Thumbnails and date will be easy to add when it will be implemented in Searx. Fix asciimoo/searx#126 --- searx/engines/flickr.py | 83 +++++++++++++++++++++++++++-------------- searx/settings.yml | 11 +++--- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py index 4ec2841dd..8b60aed1d 100644 --- a/searx/engines/flickr.py +++ b/searx/engines/flickr.py @@ -1,54 +1,83 @@ #!/usr/bin/env python +## Flickr (Images) +# +# @website https://www.flickr.com +# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, thumbnail, img_src +#More info on api-key : https://www.flickr.com/services/apps/create/ + from urllib import urlencode -#from json import loads +from json import loads from urlparse import urljoin from lxml import html from time import time categories = ['images'] -url = 'https://secure.flickr.com/' -search_url = url+'search/?{query}&page={page}' -results_xpath = '//div[@class="view display-item-tile"]/figure/div' +nb_per_page = 15 +paging = True +api_key= None + + +url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key={api_key}&{text}&sort=relevance&extras=description%2C+owner_name%2C+url_o%2C+url_z&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}' +photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' paging = True +def build_flickr_url(user_id, photo_id): + return photo_url.format(userid=user_id,photoid=photo_id) + def request(query, params): - params['url'] = search_url.format(query=urlencode({'text': query}), - page=params['pageno']) - time_string = str(int(time())-3) - params['cookies']['BX'] = '3oqjr6d9nmpgl&b=3&s=dh' - params['cookies']['xb'] = '421409' - params['cookies']['localization'] = 'en-us' - params['cookies']['flrbp'] = time_string +\ - '-3a8cdb85a427a33efda421fbda347b2eaf765a54' - params['cookies']['flrbs'] = time_string +\ - '-ed142ae8765ee62c9ec92a9513665e0ee1ba6776' - params['cookies']['flrb'] = '9' + params['url'] = url.format(text=urlencode({'text': query}), + api_key=api_key, + nb_per_page=nb_per_page, + page=params['pageno']) return params def response(resp): results = [] - dom = html.fromstring(resp.text) - for result in dom.xpath(results_xpath): - img = result.xpath('.//img') + + search_results = loads(resp.text) - if not img: + # return empty array if there are no results + if not 'photos' in search_results: + return [] + + if not 'photo' in search_results['photos']: + return [] + + photos = search_results['photos']['photo'] + + # parse results + for photo in photos: + if 'url_o' in photo: + img_src = photo['url_o'] + elif 'url_z' in photo: + img_src = photo['url_z'] + else: continue - img = img[0] - img_src = 'https:'+img.attrib.get('src') + url = build_flickr_url(photo['owner'], photo['id']) - if not img_src: - continue - - href = urljoin(url, result.xpath('.//a')[0].attrib.get('href')) - title = img.attrib.get('alt', '') - results.append({'url': href, + title = photo['title'] + + content = ''+ photo['ownername'] +'
' + + content = content + ' ' + photo['description']['_content'] + '' + + # append result + results.append({'url': url, 'title': title, 'img_src': img_src, + 'content': content, 'template': 'images.html'}) + + # return results return results diff --git a/searx/settings.yml b/searx/settings.yml index 5e9f11202..b7359b919 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -65,11 +65,12 @@ engines: # categories : files # shortcut : fc - - name : flickr - engine : flickr - categories : images - shortcut : fl - timeout: 3.0 +# api-key required: https://www.flickr.com/services/apps/create/ +# - name : flickr +# engine : flickr +# categories : images +# shortcut : fl +# api_key: 'apikey' # required! - name : general-file engine : generalfile From 0059d08f13b1bf64b3f36ab2cbe89d5fec5d727c Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Mon, 15 Dec 2014 03:21:25 +0100 Subject: [PATCH 2/3] Rework Flickr Engine Everything was redone to use the API. It needs an API key, but it's worth it. Everything works. Title, Image, Content, URL The API allow lots of things. Thumbnails and date will be easy to add when it will be implemented in Searx. Fix asciimoo/searx#126 --- searx/engines/flickr.py | 83 +++++++++++++++++++++++++++-------------- searx/settings.yml | 11 +++--- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py index 4ec2841dd..8b60aed1d 100644 --- a/searx/engines/flickr.py +++ b/searx/engines/flickr.py @@ -1,54 +1,83 @@ #!/usr/bin/env python +## Flickr (Images) +# +# @website https://www.flickr.com +# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, thumbnail, img_src +#More info on api-key : https://www.flickr.com/services/apps/create/ + from urllib import urlencode -#from json import loads +from json import loads from urlparse import urljoin from lxml import html from time import time categories = ['images'] -url = 'https://secure.flickr.com/' -search_url = url+'search/?{query}&page={page}' -results_xpath = '//div[@class="view display-item-tile"]/figure/div' +nb_per_page = 15 +paging = True +api_key= None + + +url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key={api_key}&{text}&sort=relevance&extras=description%2C+owner_name%2C+url_o%2C+url_z&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}' +photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' paging = True +def build_flickr_url(user_id, photo_id): + return photo_url.format(userid=user_id,photoid=photo_id) + def request(query, params): - params['url'] = search_url.format(query=urlencode({'text': query}), - page=params['pageno']) - time_string = str(int(time())-3) - params['cookies']['BX'] = '3oqjr6d9nmpgl&b=3&s=dh' - params['cookies']['xb'] = '421409' - params['cookies']['localization'] = 'en-us' - params['cookies']['flrbp'] = time_string +\ - '-3a8cdb85a427a33efda421fbda347b2eaf765a54' - params['cookies']['flrbs'] = time_string +\ - '-ed142ae8765ee62c9ec92a9513665e0ee1ba6776' - params['cookies']['flrb'] = '9' + params['url'] = url.format(text=urlencode({'text': query}), + api_key=api_key, + nb_per_page=nb_per_page, + page=params['pageno']) return params def response(resp): results = [] - dom = html.fromstring(resp.text) - for result in dom.xpath(results_xpath): - img = result.xpath('.//img') + + search_results = loads(resp.text) - if not img: + # return empty array if there are no results + if not 'photos' in search_results: + return [] + + if not 'photo' in search_results['photos']: + return [] + + photos = search_results['photos']['photo'] + + # parse results + for photo in photos: + if 'url_o' in photo: + img_src = photo['url_o'] + elif 'url_z' in photo: + img_src = photo['url_z'] + else: continue - img = img[0] - img_src = 'https:'+img.attrib.get('src') + url = build_flickr_url(photo['owner'], photo['id']) - if not img_src: - continue - - href = urljoin(url, result.xpath('.//a')[0].attrib.get('href')) - title = img.attrib.get('alt', '') - results.append({'url': href, + title = photo['title'] + + content = ''+ photo['ownername'] +'
' + + content = content + ' ' + photo['description']['_content'] + '' + + # append result + results.append({'url': url, 'title': title, 'img_src': img_src, + 'content': content, 'template': 'images.html'}) + + # return results return results diff --git a/searx/settings.yml b/searx/settings.yml index b51b37f1c..d2c7a32c9 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -65,11 +65,12 @@ engines: # categories : files # shortcut : fc - - name : flickr - engine : flickr - categories : images - shortcut : fl - timeout: 3.0 +# api-key required: https://www.flickr.com/services/apps/create/ +# - name : flickr +# engine : flickr +# categories : images +# shortcut : fl +# api_key: 'apikey' # required! - name : general-file engine : generalfile From 930f724ec639c167d870d716240ac5d4512beba2 Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Tue, 16 Dec 2014 20:40:03 +0100 Subject: [PATCH 3/3] Add an No Api Flickr Engine It uses the webpage json infos to build the results Let the user choose the engine in setting.yml. Noapi active by default + little corrections on Flickr engine --- searx/engines/flickr-noapi.py | 102 ++++++++++++++++++++++++++++++++++ searx/engines/flickr.py | 5 +- searx/settings.yml | 11 ++-- 3 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 searx/engines/flickr-noapi.py diff --git a/searx/engines/flickr-noapi.py b/searx/engines/flickr-noapi.py new file mode 100644 index 000000000..b44affec6 --- /dev/null +++ b/searx/engines/flickr-noapi.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +## Flickr (Images) +# +# @website https://www.flickr.com +# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html) +# +# @using-api no +# @results HTML +# @stable no +# @parse url, title, thumbnail, img_src + +from urllib import urlencode +from json import loads +from urlparse import urljoin +from lxml import html +import re + +categories = ['images'] + +url = 'https://secure.flickr.com/' +search_url = url+'search/?{query}&page={page}' +photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' +regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) + +paging = True + +def build_flickr_url(user_id, photo_id): + return photo_url.format(userid=user_id,photoid=photo_id) + + +def request(query, params): + params['url'] = search_url.format(query=urlencode({'text': query}), + page=params['pageno']) + return params + + +def response(resp): + results = [] + + matches = regex.search(resp.text) + + if matches == None: + return results + + match = matches.group(1) + search_results = loads(match) + + if not '_data' in search_results: + return [] + + photos = search_results['_data'] + + for photo in photos: + + # In paged configuration, the first pages' photos are represented by a None object + if photo == None: + continue + + # From the biggest to the lowest format + if 'o' in photo['sizes']: + img_src = photo['sizes']['o']['displayUrl'] + elif 'k' in photo['sizes']: + img_src = photo['sizes']['k']['displayUrl'] + elif 'h' in photo['sizes']: + img_src = photo['sizes']['h']['displayUrl'] + elif 'b' in photo['sizes']: + img_src = photo['sizes']['b']['displayUrl'] + elif 'c' in photo['sizes']: + img_src = photo['sizes']['c']['displayUrl'] + elif 'z' in photo['sizes']: + img_src = photo['sizes']['z']['displayUrl'] + elif 'n' in photo['sizes']: + img_src = photo['sizes']['n']['displayUrl'] + elif 'm' in photo['sizes']: + img_src = photo['sizes']['m']['displayUrl'] + elif 't' in photo['sizes']: + img_src = photo['sizes']['to']['displayUrl'] + elif 'q' in photo['sizes']: + img_src = photo['sizes']['q']['displayUrl'] + elif 's' in photo['sizes']: + img_src = photo['sizes']['s']['displayUrl'] + else: + continue + + url = build_flickr_url(photo['owner']['id'], photo['id']) + + title = photo['title'] + + content = ''+ photo['owner']['username'] +'
' + + if 'description' in photo: + content = content + '' + photo['description'] + '' + + # append result + results.append({'url': url, + 'title': title, + 'img_src': img_src, + 'content': content, + 'template': 'images.html'}) + + return results diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py index 8b60aed1d..2fa5ed7ec 100644 --- a/searx/engines/flickr.py +++ b/searx/engines/flickr.py @@ -13,9 +13,6 @@ from urllib import urlencode from json import loads -from urlparse import urljoin -from lxml import html -from time import time categories = ['images'] @@ -70,7 +67,7 @@ def response(resp): content = ''+ photo['ownername'] +'
' - content = content + ' ' + photo['description']['_content'] + '' + content = content + '' + photo['description']['_content'] + '' # append result results.append({'url': url, diff --git a/searx/settings.yml b/searx/settings.yml index d2c7a32c9..07cd2ac3e 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -65,12 +65,15 @@ engines: # categories : files # shortcut : fc -# api-key required: https://www.flickr.com/services/apps/create/ -# - name : flickr + - name : flickr + categories : images + shortcut : fl +# You can use the engine using the official stable API, but you need an API key +# See : https://www.flickr.com/services/apps/create/ # engine : flickr -# categories : images -# shortcut : fl # api_key: 'apikey' # required! +# Or you can use the html non-stable engine, activated by default + engine : flickr-noapi - name : general-file engine : generalfile