From 7c4e4ebd40d342735bf2e207d469aed2ca25cfa9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 23 Nov 2024 11:18:49 +0100 Subject: [PATCH] [log] warning with URL in case of 'raise_for_httperror' In order to be able to implement error handling, it is necessary to know which URL triggered the exception / the URL has not yet been logged. Signed-off-by: Markus Heiser --- searx/network/network.py | 12 +++++++----- searx/search/processors/online.py | 3 --- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/searx/network/network.py b/searx/network/network.py index 453c8d2fc..ff02d447a 100644 --- a/searx/network/network.py +++ b/searx/network/network.py @@ -233,8 +233,7 @@ class Network: del kwargs['raise_for_httperror'] return do_raise_for_httperror - @staticmethod - def patch_response(response, do_raise_for_httperror): + def patch_response(self, response, do_raise_for_httperror): if isinstance(response, httpx.Response): # requests compatibility (response is not streamed) # see also https://www.python-httpx.org/compatibility/#checking-for-4xx5xx-responses @@ -242,8 +241,11 @@ class Network: # raise an exception if do_raise_for_httperror: - raise_for_httperror(response) - + try: + raise_for_httperror(response) + except: + self._logger.warning(f"HTTP Request failed: {response.request.method} {response.request.url}") + raise return response def is_valid_response(self, response): @@ -269,7 +271,7 @@ class Network: else: response = await client.request(method, url, **kwargs) if self.is_valid_response(response) or retries <= 0: - return Network.patch_response(response, do_raise_for_httperror) + return self.patch_response(response, do_raise_for_httperror) except httpx.RemoteProtocolError as e: if not was_disconnected: # the server has closed the connection: diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index acc3792f5..778b4ac4d 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -137,9 +137,6 @@ class OnlineProcessor(EngineProcessor): self.engine.request(query, params) # ignoring empty urls - if params['url'] is None: - return None - if not params['url']: return None