diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 5c641ab0e..6d5066733 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -40,7 +40,9 @@ engine_default_args = {'paging': False, 'safesearch': False, 'timeout': settings['outgoing']['request_timeout'], 'shortcut': '-', - 'disabled': False} + 'disabled': False, + 'suspend_end_time': 0, + 'continuous_errors': 0} def load_module(filename): diff --git a/searx/search.py b/searx/search.py index 655b7808a..ce41b231b 100644 --- a/searx/search.py +++ b/searx/search.py @@ -34,16 +34,23 @@ number_of_searches = 0 def search_request_wrapper(fn, url, engine_name, **kwargs): + ret = None + engine = engines[engine_name] try: - return fn(url, **kwargs) + ret = fn(url, **kwargs) + with threading.RLock(): + engine.continuous_errors = 0 + engine.suspend_end_time = 0 except: # increase errors stats with threading.RLock(): - engines[engine_name].stats['errors'] += 1 + engine.stats['errors'] += 1 + engine.continuous_errors += 1 + engine.suspend_end_time = time() + min(60, engine.continuous_errors) # print engine name and specific error message logger.exception('engine crash: {0}'.format(engine_name)) - return + return ret def threaded_requests(requests): @@ -241,6 +248,10 @@ class Search(object): for engine in categories[categ] if (engine.name, categ) not in self.blocked_engines) + # remove suspended engines + self.engines = [e for e in self.engines + if engines[e['name']].suspend_end_time <= time()] + # do search-request def search(self, request): global number_of_searches