jandieman
/
searxng
mirror of https://github.com/searxng/searxng.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

[fix] pep8 : engines (errors E121, E127, E128 and E501 still exist)

This commit is contained in:
dalf 2014-12-07 16:37:56 +01:00
parent ffcec383b7
commit 7c13d630e4
22 changed files with 109 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
searx/engines/bing.py
View File

@ -1,8 +1,9 @@
## Bing (Web) ## Bing (Web)
# #
# @website https://www.bing.com # @website https://www.bing.com
# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month # @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
# # max. 5000 query/month
#
# @using-api no (because of query limit) # @using-api no (because of query limit)
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable no (HTML can change) # @stable no (HTML can change)
@ -58,8 +59,8 @@ def response(resp):
content = escape(' '.join(result.xpath('.//p//text()'))) content = escape(' '.join(result.xpath('.//p//text()')))
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,
'content': content}) 'content': content})
# return results if something is found # return results if something is found
@ -74,8 +75,8 @@ def response(resp):
content = escape(' '.join(result.xpath('.//p//text()'))) content = escape(' '.join(result.xpath('.//p//text()')))
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,
'content': content}) 'content': content})
# return results # return results

20
searx/engines/bing_images.py
View File

@ -1,17 +1,19 @@
## Bing (Images) ## Bing (Images)
# #
# @website https://www.bing.com/images # @website https://www.bing.com/images
# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month # @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
# # max. 5000 query/month
#
# @using-api no (because of query limit) # @using-api no (because of query limit)
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable no (HTML can change) # @stable no (HTML can change)
# @parse url, title, img_src # @parse url, title, img_src
# #
# @todo currently there are up to 35 images receive per page, because bing does not parse count=10. limited response to 10 images # @todo currently there are up to 35 images receive per page,
# because bing does not parse count=10.
# limited response to 10 images
from urllib import urlencode from urllib import urlencode
from cgi import escape
from lxml import html from lxml import html
from yaml import load from yaml import load
import re import re
@ -51,15 +53,15 @@ def response(resp):
dom = html.fromstring(resp.content) dom = html.fromstring(resp.content)
# init regex for yaml-parsing # init regex for yaml-parsing
p = re.compile( '({|,)([a-z]+):(")') p = re.compile('({|,)([a-z]+):(")')
# parse results # parse results
for result in dom.xpath('//div[@class="dg_u"]'): for result in dom.xpath('//div[@class="dg_u"]'):
link = result.xpath('./a')[0] link = result.xpath('./a')[0]
# parse yaml-data (it is required to add a space, to make it parsable) # parse yaml-data (it is required to add a space, to make it parsable)
yaml_data = load(p.sub( r'\1\2: \3', link.attrib.get('m'))) yaml_data = load(p.sub(r'\1\2: \3', link.attrib.get('m')))
title = link.attrib.get('t1') title = link.attrib.get('t1')
#url = 'http://' + link.attrib.get('t3') #url = 'http://' + link.attrib.get('t3')
url = yaml_data.get('surl') url = yaml_data.get('surl')
@ -69,7 +71,7 @@ def response(resp):
results.append({'template': 'images.html', results.append({'template': 'images.html',
'url': url, 'url': url,
'title': title, 'title': title,
'content': '', 'content': '',
'img_src': img_src}) 'img_src': img_src})
# TODO stop parsing if 10 images are found # TODO stop parsing if 10 images are found

19
searx/engines/bing_news.py
View File

@ -1,8 +1,9 @@
## Bing (News) ## Bing (News)
# #
# @website https://www.bing.com/news # @website https://www.bing.com/news
# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month # @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
# # max. 5000 query/month
#
# @using-api no (because of query limit) # @using-api no (because of query limit)
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable no (HTML can change) # @stable no (HTML can change)
@ -57,12 +58,12 @@ def response(resp):
url = link.attrib.get('href') url = link.attrib.get('href')
title = ' '.join(link.xpath('.//text()')) title = ' '.join(link.xpath('.//text()'))
contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()') contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()')
if contentXPath != None: if contentXPath is not None:
content = escape(' '.join(contentXPath)) content = escape(' '.join(contentXPath))
# parse publishedDate # parse publishedDate
publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()') publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()')
if publishedDateXPath != None: if publishedDateXPath is not None:
publishedDate = escape(' '.join(publishedDateXPath)) publishedDate = escape(' '.join(publishedDateXPath))
if re.match("^[0-9]+ minute(s|) ago$", publishedDate): if re.match("^[0-9]+ minute(s|) ago$", publishedDate):
@ -89,10 +90,10 @@ def response(resp):
except TypeError: except TypeError:
# FIXME # FIXME
publishedDate = datetime.now() publishedDate = datetime.now()
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,
'publishedDate': publishedDate, 'publishedDate': publishedDate,
'content': content}) 'content': content})

2
searx/engines/currency_convert.py
View File

@ -55,6 +55,6 @@ def response(resp):
resp.search_params['to'].lower() resp.search_params['to'].lower()
) )
results.append({'answer' : answer, 'url': url}) results.append({'answer': answer, 'url': url})
return results return results

5
searx/engines/dailymotion.py
View File

@ -1,8 +1,8 @@
## Dailymotion (Videos) ## Dailymotion (Videos)
# #
# @website https://www.dailymotion.com # @website https://www.dailymotion.com
# @provide-api yes (http://www.dailymotion.com/developer) # @provide-api yes (http://www.dailymotion.com/developer)
# #
# @using-api yes # @using-api yes
# @results JSON # @results JSON
# @stable yes # @stable yes
@ -12,7 +12,6 @@
from urllib import urlencode from urllib import urlencode
from json import loads from json import loads
from lxml import html
# engine dependent config # engine dependent config
categories = ['videos'] categories = ['videos']

4
searx/engines/deviantart.py
View File

@ -1,8 +1,8 @@
## Deviantart (Images) ## Deviantart (Images)
# #
# @website https://www.deviantart.com/ # @website https://www.deviantart.com/
# @provide-api yes (https://www.deviantart.com/developers/) (RSS) # @provide-api yes (https://www.deviantart.com/developers/) (RSS)
# #
# @using-api no (TODO, rewrite to api) # @using-api no (TODO, rewrite to api)
# @results HTML # @results HTML
# @stable no (HTML can change) # @stable no (HTML can change)

12
searx/engines/duckduckgo.py
View File

@ -1,15 +1,17 @@
## DuckDuckGo (Web) ## DuckDuckGo (Web)
# #
# @website https://duckduckgo.com/ # @website https://duckduckgo.com/
# @provide-api yes (https://duckduckgo.com/api), but not all results from search-site # @provide-api yes (https://duckduckgo.com/api),
# # but not all results from search-site
#
# @using-api no # @using-api no
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable no (HTML can change) # @stable no (HTML can change)
# @parse url, title, content # @parse url, title, content
# #
# @todo rewrite to api # @todo rewrite to api
# @todo language support (the current used site does not support language-change) # @todo language support
# (the current used site does not support language-change)
from urllib import urlencode from urllib import urlencode
from lxml.html import fromstring from lxml.html import fromstring
@ -37,7 +39,7 @@ def request(query, params):
if params['language'] == 'all': if params['language'] == 'all':
locale = 'en-us' locale = 'en-us'
else: else:
locale = params['language'].replace('_','-').lower() locale = params['language'].replace('_', '-').lower()
params['url'] = url.format( params['url'] = url.format(
query=urlencode({'q': query, 'kl': locale}), query=urlencode({'q': query, 'kl': locale}),

2
searx/engines/dummy.py
View File

@ -1,5 +1,5 @@
## Dummy ## Dummy
# #
# @results empty array # @results empty array
# @stable yes # @stable yes

11
searx/engines/faroo.py
View File

@ -1,8 +1,8 @@
## Faroo (Web, News) ## Faroo (Web, News)
# #
# @website http://www.faroo.com # @website http://www.faroo.com
# @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key # @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key
# #
# @using-api yes # @using-api yes
# @results JSON # @results JSON
# @stable yes # @stable yes
@ -24,9 +24,10 @@ api_key = None
url = 'http://www.faroo.com/' url = 'http://www.faroo.com/'
search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}' search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}'
search_category = {'general': 'web', search_category = {'general': 'web',
'news': 'news'} 'news': 'news'}
# do search-request # do search-request
def request(query, params): def request(query, params):
offset = (params['pageno']-1) * number_of_results + 1 offset = (params['pageno']-1) * number_of_results + 1
@ -48,7 +49,7 @@ def request(query, params):
query=urlencode({'q': query}), query=urlencode({'q': query}),
language=language, language=language,
categorie=categorie, categorie=categorie,
api_key=api_key ) api_key=api_key)
# using searx User-Agent # using searx User-Agent
params['headers']['User-Agent'] = searx_useragent() params['headers']['User-Agent'] = searx_useragent()
@ -101,7 +102,7 @@ def response(resp):
results.append({'template': 'images.html', results.append({'template': 'images.html',
'url': result['url'], 'url': result['url'],
'title': result['title'], 'title': result['title'],
'content': result['kwic'], 'content': result['kwic'],
'img_src': result['iurl']}) 'img_src': result['iurl']})
# return results # return results

4
searx/engines/generalfile.py
View File

@ -1,8 +1,8 @@
## General Files (Files) ## General Files (Files)
# #
# @website http://www.general-files.org # @website http://www.general-files.org
# @provide-api no (nothing found) # @provide-api no (nothing found)
# #
# @using-api no (because nothing found) # @using-api no (because nothing found)
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable no (HTML can change) # @stable no (HTML can change)

4
searx/engines/github.py
View File

@ -1,8 +1,8 @@
## Github (It) ## Github (It)
# #
# @website https://github.com/ # @website https://github.com/
# @provide-api yes (https://developer.github.com/v3/) # @provide-api yes (https://developer.github.com/v3/)
# #
# @using-api yes # @using-api yes
# @results JSON # @results JSON
# @stable yes (using api) # @stable yes (using api)

7
searx/engines/google_images.py
View File

@ -1,8 +1,9 @@
## Google (Images) ## Google (Images)
# #
# @website https://www.google.com # @website https://www.google.com
# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! # @provide-api yes (https://developers.google.com/web-search/docs/),
# # deprecated!
#
# @using-api yes # @using-api yes
# @results JSON # @results JSON
# @stable yes (but deprecated) # @stable yes (but deprecated)

7
searx/engines/google_news.py
View File

@ -1,8 +1,9 @@
## Google (News) ## Google (News)
# #
# @website https://www.google.com # @website https://www.google.com
# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated! # @provide-api yes (https://developers.google.com/web-search/docs/),
# # deprecated!
#
# @using-api yes # @using-api yes
# @results JSON # @results JSON
# @stable yes (but deprecated) # @stable yes (but deprecated)

30
searx/engines/openstreetmap.py
View File

@ -39,16 +39,16 @@ def response(resp):
url = result_base_url.format(osm_type=osm_type, url = result_base_url.format(osm_type=osm_type,
osm_id=r['osm_id']) osm_id=r['osm_id'])
osm = {'type':osm_type, osm = {'type': osm_type,
'id':r['osm_id']} 'id': r['osm_id']}
geojson = r.get('geojson') geojson = r.get('geojson')
# if no geojson is found and osm_type is a node, add geojson Point # if no geojson is found and osm_type is a node, add geojson Point
if not geojson and\ if not geojson and\
osm_type == 'node': osm_type == 'node':
geojson = {u'type':u'Point', geojson = {u'type': u'Point',
u'coordinates':[r['lon'],r['lat']]} u'coordinates': [r['lon'], r['lat']]}
address_raw = r.get('address') address_raw = r.get('address')
address = {} address = {}
@ -59,20 +59,20 @@ def response(resp):
r['class'] == 'tourism' or\ r['class'] == 'tourism' or\
r['class'] == 'leisure': r['class'] == 'leisure':
if address_raw.get('address29'): if address_raw.get('address29'):
address = {'name':address_raw.get('address29')} address = {'name': address_raw.get('address29')}
else: else:
address = {'name':address_raw.get(r['type'])} address = {'name': address_raw.get(r['type'])}
# add rest of adressdata, if something is already found # add rest of adressdata, if something is already found
if address.get('name'): if address.get('name'):
address.update({'house_number':address_raw.get('house_number'), address.update({'house_number': address_raw.get('house_number'),
'road':address_raw.get('road'), 'road': address_raw.get('road'),
'locality':address_raw.get('city', 'locality': address_raw.get('city',
address_raw.get('town', address_raw.get('town',
address_raw.get('village'))), address_raw.get('village'))),
'postcode':address_raw.get('postcode'), 'postcode': address_raw.get('postcode'),
'country':address_raw.get('country'), 'country': address_raw.get('country'),
'country_code':address_raw.get('country_code')}) 'country_code': address_raw.get('country_code')})
else: else:
address = None address = None

6
searx/engines/piratebay.py
View File

@ -1,8 +1,8 @@
## Piratebay (Videos, Music, Files) ## Piratebay (Videos, Music, Files)
# #
# @website https://thepiratebay.se # @website https://thepiratebay.se
# @provide-api no (nothing found) # @provide-api no (nothing found)
# #
# @using-api no # @using-api no
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable yes (HTML can change) # @stable yes (HTML can change)
@ -23,7 +23,7 @@ url = 'https://thepiratebay.se/'
search_url = url + 'search/{search_term}/{pageno}/99/{search_type}' search_url = url + 'search/{search_term}/{pageno}/99/{search_type}'
# piratebay specific type-definitions # piratebay specific type-definitions
search_types = {'files': '0', search_types = {'files': '0',
'music': '100', 'music': '100',
'videos': '200'} 'videos': '200'}

4
searx/engines/soundcloud.py
View File

@ -1,8 +1,8 @@
## Soundcloud (Music) ## Soundcloud (Music)
# #
# @website https://soundcloud.com # @website https://soundcloud.com
# @provide-api yes (https://developers.soundcloud.com/) # @provide-api yes (https://developers.soundcloud.com/)
# #
# @using-api yes # @using-api yes
# @results JSON # @results JSON
# @stable yes # @stable yes

8
searx/engines/stackoverflow.py
View File

@ -1,8 +1,8 @@
## Stackoverflow (It) ## Stackoverflow (It)
# #
# @website https://stackoverflow.com/ # @website https://stackoverflow.com/
# @provide-api not clear (https://api.stackexchange.com/docs/advanced-search) # @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)
# #
# @using-api no # @using-api no
# @results HTML # @results HTML
# @stable no (HTML can change) # @stable no (HTML can change)
@ -50,8 +50,8 @@ def response(resp):
content = escape(' '.join(result.xpath(content_xpath))) content = escape(' '.join(result.xpath(content_xpath)))
# append result # append result
results.append({'url': href, results.append({'url': href,
'title': title, 'title': title,
'content': content}) 'content': content})
# return results # return results

4
searx/engines/twitter.py
View File

@ -1,8 +1,8 @@
## Twitter (Social media) ## Twitter (Social media)
# #
# @website https://www.bing.com/news # @website https://www.bing.com/news
# @provide-api yes (https://dev.twitter.com/docs/using-search) # @provide-api yes (https://dev.twitter.com/docs/using-search)
# #
# @using-api no # @using-api no
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable no (HTML can change) # @stable no (HTML can change)

12
searx/engines/vimeo.py
View File

@ -1,8 +1,9 @@
## Vimeo (Videos) ## Vimeo (Videos)
# #
# @website https://vimeo.com/ # @website https://vimeo.com/
# @provide-api yes (http://developer.vimeo.com/api), they have a maximum count of queries/hour # @provide-api yes (http://developer.vimeo.com/api),
# # they have a maximum count of queries/hour
#
# @using-api no (TODO, rewrite to api) # @using-api no (TODO, rewrite to api)
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable no (HTML can change) # @stable no (HTML can change)
@ -35,11 +36,12 @@ publishedDate_xpath = './/p[@class="meta"]//attribute::datetime'
# do search-request # do search-request
def request(query, params): def request(query, params):
params['url'] = search_url.format(pageno=params['pageno'] , params['url'] = search_url.format(pageno=params['pageno'],
query=urlencode({'q': query})) query=urlencode({'q': query}))
# TODO required? # TODO required?
params['cookies']['__utma'] = '00000000.000#0000000.0000000000.0000000000.0000000000.0' params['cookies']['__utma'] =\
'00000000.000#0000000.0000000000.0000000000.0000000000.0'
return params return params

11
searx/engines/yacy.py
View File

@ -1,8 +1,9 @@
## Yacy (Web, Images, Videos, Music, Files) ## Yacy (Web, Images, Videos, Music, Files)
# #
# @website http://yacy.net # @website http://yacy.net
# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch) # @provide-api yes
# # (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
#
# @using-api yes # @using-api yes
# @results JSON # @results JSON
# @stable yes # @stable yes
@ -16,7 +17,7 @@ from urllib import urlencode
from dateutil import parser from dateutil import parser
# engine dependent config # engine dependent config
categories = ['general', 'images'] #TODO , 'music', 'videos', 'files' categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'
paging = True paging = True
language_support = True language_support = True
number_of_results = 5 number_of_results = 5
@ -28,7 +29,7 @@ search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limi
# yacy specific type-definitions # yacy specific type-definitions
search_types = {'general': 'text', search_types = {'general': 'text',
'images': 'image', 'images': 'image',
'files': 'app', 'files': 'app',
'music': 'audio', 'music': 'audio',
'videos': 'video'} 'videos': 'video'}

15
searx/engines/yahoo.py
View File

@ -1,8 +1,9 @@
## Yahoo (Web) ## Yahoo (Web)
# #
# @website https://search.yahoo.com/web # @website https://search.yahoo.com/web
# @provide-api yes (https://developer.yahoo.com/boss/search/), $0.80/1000 queries # @provide-api yes (https://developer.yahoo.com/boss/search/),
# # $0.80/1000 queries
#
# @using-api no (because pricing) # @using-api no (because pricing)
# @results HTML (using search portal) # @results HTML (using search portal)
# @stable no (HTML can change) # @stable no (HTML can change)
@ -40,8 +41,8 @@ def parse_url(url_string):
if endpos > -1: if endpos > -1:
endpositions.append(endpos) endpositions.append(endpos)
if start==0 or len(endpositions) == 0: if start == 0 or len(endpositions) == 0:
return url_string return url_string
else: else:
end = min(endpositions) end = min(endpositions)
return unquote(url_string[start:end]) return unquote(url_string[start:end])
@ -84,8 +85,8 @@ def response(resp):
content = extract_text(result.xpath(content_xpath)[0]) content = extract_text(result.xpath(content_xpath)[0])
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,
'content': content}) 'content': content})
# if no suggestion found, return results # if no suggestion found, return results

4
searx/engines/youtube.py
View File

@ -1,8 +1,8 @@
## Youtube (Videos) ## Youtube (Videos)
# #
# @website https://www.youtube.com/ # @website https://www.youtube.com/
# @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/) # @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/)
# #
# @using-api yes # @using-api yes
# @results JSON # @results JSON
# @stable yes # @stable yes