jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2025-12-22 19:50:00 +00:00
Code Issues Projects Releases Wiki Activity

[mod] multithreading only in searx.search.* packages

Browse Source 
it prepares the new architecture change,
everything about multithreading in moved in the searx.search.* packages

previously the call to the "init" function of the engines was done in searx.engines:
* the network was not set (request not sent using the defined proxy)
* it requires to monkey patch the code to avoid HTTP requests during the tests
This commit is contained in:
Alexandre Flament
2021-05-05 13:08:54 +02:00
parent d36adfa59f
commit 8c1a65d32f
10 changed files with 85 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
searx/search/processors/online.py
View File

@@ -5,7 +5,7 @@
"""
from time import time
from timeit import default_timer
import asyncio
import httpx
@@ -40,6 +40,15 @@ class OnlineProcessor(EngineProcessor):
engine_type = 'online'
def initialize(self):
# set timeout for all HTTP requests
searx.network.set_timeout_for_thread(self.engine.timeout, start_time=default_timer())
# reset the HTTP total time
searx.network.reset_time_for_thread()
# set the network
searx.network.set_context_network_name(self.engine_name)
super().initialize()
def get_params(self, search_query, engine_category):
params = super().get_params(search_query, engine_category)
if params is None:
@@ -139,7 +148,7 @@ class OnlineProcessor(EngineProcessor):
self.handle_exception(result_container, e, suspend=True)
logger.error("engine {0} : HTTP requests timeout"
"(search duration : {1} s, timeout: {2} s) : {3}"
.format(self.engine_name, time() - start_time,
.format(self.engine_name, default_timer() - start_time,
timeout_limit,
e.__class__.__name__))
except (httpx.HTTPError, httpx.StreamError) as e:
@@ -147,7 +156,7 @@ class OnlineProcessor(EngineProcessor):
self.handle_exception(result_container, e, suspend=True)
logger.exception("engine {0} : requests exception"
"(search duration : {1} s, timeout: {2} s) : {3}"
.format(self.engine_name, time() - start_time,
.format(self.engine_name, default_timer() - start_time,
timeout_limit,
e))
except SearxEngineCaptchaException as e: