mirror of
https://github.com/searxng/searxng.git
synced 2025-12-22 19:50:00 +00:00
[mod] multithreading only in searx.search.* packages
it prepares the new architecture change, everything about multithreading in moved in the searx.search.* packages previously the call to the "init" function of the engines was done in searx.engines: * the network was not set (request not sent using the defined proxy) * it requires to monkey patch the code to avoid HTTP requests during the tests
This commit is contained in:
@@ -11,9 +11,11 @@ __all__ = [
|
||||
'OnlineProcessor',
|
||||
'OnlineDictionaryProcessor',
|
||||
'OnlineCurrencyProcessor',
|
||||
'processors',
|
||||
'PROCESSORS',
|
||||
]
|
||||
|
||||
import threading
|
||||
|
||||
from searx import logger
|
||||
import searx.engines as engines
|
||||
|
||||
@@ -24,7 +26,7 @@ from .online_currency import OnlineCurrencyProcessor
|
||||
from .abstract import EngineProcessor
|
||||
|
||||
logger = logger.getChild('search.processors')
|
||||
processors = {}
|
||||
PROCESSORS = {}
|
||||
"""Cache request processores, stored by *engine-name* (:py:func:`initialize`)"""
|
||||
|
||||
def get_processor_class(engine_type):
|
||||
@@ -34,6 +36,7 @@ def get_processor_class(engine_type):
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def get_processor(engine, engine_name):
|
||||
"""Return processor instance that fits to ``engine.engine.type``)"""
|
||||
engine_type = getattr(engine, 'engine_type', 'online')
|
||||
@@ -42,12 +45,26 @@ def get_processor(engine, engine_name):
|
||||
return processor_class(engine, engine_name)
|
||||
return None
|
||||
|
||||
|
||||
def initialize_processor(processor):
|
||||
"""Initialize one processor
|
||||
|
||||
Call the init function of the engine
|
||||
"""
|
||||
if processor.has_initialize_function:
|
||||
t = threading.Thread(target=processor.initialize, daemon=True)
|
||||
t.start()
|
||||
|
||||
|
||||
def initialize(engine_list):
|
||||
"""Initialize all engines and store a processor for each engine in :py:obj:`processors`."""
|
||||
engines.initialize_engines(engine_list)
|
||||
for engine_name, engine in engines.engines.items():
|
||||
processor = get_processor(engine, engine_name)
|
||||
if processor is None:
|
||||
logger.error('Error get processor for engine %s', engine_name)
|
||||
else:
|
||||
processors[engine_name] = processor
|
||||
"""Initialize all engines and store a processor for each engine in :py:obj:`PROCESSORS`."""
|
||||
for engine_data in engine_list:
|
||||
engine_name = engine_data['name']
|
||||
engine = engines.engines.get(engine_name)
|
||||
if engine:
|
||||
processor = get_processor(engine, engine_name)
|
||||
initialize_processor(processor)
|
||||
if processor is None:
|
||||
logger.error('Error get processor for engine %s', engine_name)
|
||||
else:
|
||||
PROCESSORS[engine_name] = processor
|
||||
|
||||
Reference in New Issue
Block a user