jandieman
/
searxng
mirror of https://github.com/searxng/searxng.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

[enh] suspend engines after error 

The duration is based on the number of continuous errors, but maximized in one minute
This commit is contained in:
Adam Tauber 2016-02-20 00:21:56 +01:00
parent 17b0c9f74a
commit 10947536aa
2 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
searx/engines/__init__.py
View File

@ -40,7 +40,9 @@ engine_default_args = {'paging': False,
'safesearch': False, 'safesearch': False,
'timeout': settings['outgoing']['request_timeout'], 'timeout': settings['outgoing']['request_timeout'],
'shortcut': '-', 'shortcut': '-',
'disabled': False} 'disabled': False,
'suspend_end_time': 0,
'continuous_errors': 0}
def load_module(filename): def load_module(filename):

17
searx/search.py
View File

@ -34,16 +34,23 @@ number_of_searches = 0
def search_request_wrapper(fn, url, engine_name, **kwargs): def search_request_wrapper(fn, url, engine_name, **kwargs):
ret = None
engine = engines[engine_name]
try: try:
return fn(url, **kwargs) ret = fn(url, **kwargs)
with threading.RLock():
engine.continuous_errors = 0
engine.suspend_end_time = 0
except: except:
# increase errors stats # increase errors stats
with threading.RLock(): with threading.RLock():
engines[engine_name].stats['errors'] += 1 engine.stats['errors'] += 1
engine.continuous_errors += 1
engine.suspend_end_time = time() + min(60, engine.continuous_errors)
# print engine name and specific error message # print engine name and specific error message
logger.exception('engine crash: {0}'.format(engine_name)) logger.exception('engine crash: {0}'.format(engine_name))
return return ret
def threaded_requests(requests): def threaded_requests(requests):
@ -241,6 +248,10 @@ class Search(object):
for engine in categories[categ] for engine in categories[categ]
if (engine.name, categ) not in self.blocked_engines) if (engine.name, categ) not in self.blocked_engines)
# remove suspended engines
self.engines = [e for e in self.engines
if engines[e['name']].suspend_end_time <= time()]
# do search-request # do search-request
def search(self, request): def search(self, request):
global number_of_searches global number_of_searches