jandieman/searxng
2
0
Fork 0
mirror of https://github.com/searxng/searxng.git synced 2025-12-23 04:00:02 +00:00
Code Issues Projects Releases Wiki Activity

[mod] ExpireCacheSQLite - implement .setmany() for bulk loading

Browse Source 
[1] https://github.com/searxng/searxng/issues/5223#issuecomment-3328597147

Suggested-by: Ivan G <igabaldon@inetol.net> [1]
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
2025-09-24 19:34:03 +02:00
committed by Markus Heiser
parent 4f4de3fc87
commit 18a58943cc
4 changed files with 131 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
searx/data/tracker_patterns.py
View File

@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Simple implementation to store TrackerPatterns data in a SQL database."""
import typing
import typing as t
__all__ = ["TrackerPatternsDB"]
@@ -14,9 +14,14 @@ from httpx import HTTPError
from searx.data.core import get_cache, log
from searx.network import get as http_get
if t.TYPE_CHECKING:
from searx.cache import CacheRowType
RuleType = tuple[str, list[str], list[str]]
@t.final
class TrackerPatternsDB:
# pylint: disable=missing-class-docstring
@@ -31,9 +36,9 @@ class TrackerPatternsDB:
class Fields:
# pylint: disable=too-few-public-methods, invalid-name
url_regexp: typing.Final = 0 # URL (regular expression) match condition of the link
url_ignore: typing.Final = 1 # URL (regular expression) to ignore
del_args: typing.Final = 2 # list of URL arguments (regular expression) to delete
url_regexp: t.Final = 0 # URL (regular expression) match condition of the link
url_ignore: t.Final = 1 # URL (regular expression) to ignore
del_args: t.Final = 2 # list of URL arguments (regular expression) to delete
def __init__(self):
self.cache = get_cache()
@@ -49,19 +54,25 @@ class TrackerPatternsDB:
def load(self):
log.debug("init searx.data.TRACKER_PATTERNS")
for rule in self.iter_clear_list():
self.add(rule)
rows: "list[CacheRowType]" = []
def add(self, rule: RuleType):
self.cache.set(
key=rule[self.Fields.url_regexp],
value=(
for rule in self.iter_clear_list():
key = rule[self.Fields.url_regexp]
value = (
rule[self.Fields.url_ignore],
rule[self.Fields.del_args],
),
ctx=self.ctx_name,
expire=None,
)
rows.append((key, value, None))
self.cache.setmany(rows, ctx=self.ctx_name)
def add(self, rule: RuleType):
key = rule[self.Fields.url_regexp]
value = (
rule[self.Fields.url_ignore],
rule[self.Fields.del_args],
)
self.cache.set(key=key, value=value, ctx=self.ctx_name, expire=None)
def rules(self) -> Iterator[RuleType]:
self.init()