mirror of
https://github.com/searxng/searxng.git
synced 2026-07-17 21:41:24 +00:00
Compare commits
15 Commits
c19d86faa3
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
81c9c23862 | ||
|
|
6913fba208 | ||
|
|
2daa4d4815 | ||
|
|
de8f73f434 | ||
|
|
9f9c00819e | ||
|
|
b72a87676f | ||
|
|
f2432e33d6 | ||
|
|
7b2199ecdf | ||
|
|
4a9c19d7bf | ||
|
|
5cb4cb2bc5 | ||
|
|
5a448596ab | ||
|
|
9c49b7e0d7 | ||
|
|
58e02a01ae | ||
|
|
7fa9f16225 | ||
|
|
9e25585aec |
2
.github/workflows/integration.yml
vendored
2
.github/workflows/integration.yml
vendored
@@ -67,7 +67,7 @@ jobs:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: "26"
|
||||
check-latest: "true"
|
||||
|
||||
@@ -4,7 +4,7 @@ cov-core==1.15.0
|
||||
black==25.9.0
|
||||
pylint==4.0.6
|
||||
splinter==0.21.0
|
||||
selenium==4.45.0
|
||||
selenium==4.46.0
|
||||
Sphinx==8.2.3;python_version <= "3.11"
|
||||
Sphinx==9.1.0; python_version > "3.11"
|
||||
sphinx-issues==6.0.0
|
||||
|
||||
@@ -13,7 +13,7 @@ sniffio==1.3.1
|
||||
valkey==6.1.1
|
||||
markdown-it-py==4.2.0
|
||||
msgspec==0.21.1
|
||||
typer==0.26.8
|
||||
typer==0.27.0
|
||||
isodate==0.7.2
|
||||
whitenoise==6.12.0
|
||||
typing-extensions==4.16.0
|
||||
|
||||
@@ -48,7 +48,7 @@ class ExpireCacheCfg(msgspec.Struct): # pylint: disable=too-few-public-methods
|
||||
MAXHOLD_TIME: int = 60 * 60 * 24 * 7 # 7 days
|
||||
"""Hold time (default in sec.), after which a value is removed from the cache."""
|
||||
|
||||
MAINTENANCE_PERIOD: int = 60 * 60 # 2h
|
||||
MAINTENANCE_PERIOD: int = 60 * 60 # 1h
|
||||
"""Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to
|
||||
``auto``."""
|
||||
|
||||
@@ -458,12 +458,22 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache):
|
||||
# Before values are taken from the table, a maintenance interval may
|
||||
# need to be carried out.
|
||||
self.maintenance()
|
||||
sql = f"SELECT value FROM {table} WHERE key = ?"
|
||||
sql = f"SELECT value, expire FROM {table} WHERE key = ?"
|
||||
row = self.DB.execute(sql, (key,)).fetchone()
|
||||
if row is None:
|
||||
return default
|
||||
|
||||
return self.deserialize(row[0])
|
||||
# Check if value is expired. It's possible that it's expired but has not
|
||||
# yet been automatically deleted by the periodic maintenance
|
||||
(value, expire) = row
|
||||
now = time.time()
|
||||
if expire < now:
|
||||
# The record is deleted during the maintenance interval. Deleting
|
||||
# the record at this point offers no advantage, as a SELECT
|
||||
# statement must be executed for every cache.get request anyways.
|
||||
return default
|
||||
|
||||
return self.deserialize(value)
|
||||
|
||||
def pairs(self, ctx: str) -> Iterator[tuple[str, typing.Any]]:
|
||||
"""Iterate over key/value pairs from table given by argument ``ctx``.
|
||||
|
||||
@@ -47,7 +47,7 @@ ENGINES_CACHE: ExpireCacheSQLite = ExpireCacheSQLite.build_cache(
|
||||
ExpireCacheCfg(
|
||||
name="ENGINES_CACHE",
|
||||
MAXHOLD_TIME=60 * 60 * 24 * 7, # 7 days
|
||||
MAINTENANCE_PERIOD=60 * 60, # 2h
|
||||
MAINTENANCE_PERIOD=60 * 60, # 1h
|
||||
MAX_VALUE_LEN=1024 * 1024 * 1024, # 1MB
|
||||
)
|
||||
)
|
||||
|
||||
@@ -83,7 +83,7 @@ def extract_video_data(video_block):
|
||||
published_date = None
|
||||
if create_time:
|
||||
try:
|
||||
published_date = datetime.strptime(create_time.strip(), "%Y-%m-%d")
|
||||
published_date = datetime.fromisoformat(create_time.strip())
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
comments_elements = eval_xpath_getindex(entry, xpath_comment, 0, default=None)
|
||||
comments: str = "" if comments_elements is None else comments_elements.text
|
||||
|
||||
publishedDate = datetime.strptime(eval_xpath_getindex(entry, xpath_published, 0).text, "%Y-%m-%dT%H:%M:%SZ")
|
||||
publishedDate = datetime.fromisoformat(eval_xpath_getindex(entry, xpath_published, 0).text.rstrip("Z"))
|
||||
|
||||
res.add(
|
||||
res.types.Paper(
|
||||
|
||||
@@ -186,7 +186,7 @@ def parse_images(data):
|
||||
img_date = item.get("bdImgnewsDate")
|
||||
publishedDate = None
|
||||
if img_date:
|
||||
publishedDate = datetime.strptime(img_date, "%Y-%m-%d %H:%M")
|
||||
publishedDate = datetime.fromisoformat(img_date)
|
||||
results.append(
|
||||
{
|
||||
"template": "images.html",
|
||||
|
||||
@@ -44,7 +44,7 @@ def response(resp):
|
||||
"url": 'https://www.bitchute.com/video/' + item['video_id'],
|
||||
"content": html_to_text(item['description']),
|
||||
"author": item['channel']['channel_name'],
|
||||
"publishedDate": datetime.strptime(item["date_published"], "%Y-%m-%dT%H:%M:%S.%fZ"),
|
||||
"publishedDate": datetime.fromisoformat(item["date_published"].rstrip("Z")),
|
||||
"length": item['duration'],
|
||||
"views": item['view_count'],
|
||||
"thumbnail": item['thumbnail_url'],
|
||||
|
||||
@@ -104,7 +104,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
title=_remove_keyword_marker(result["Subject"]),
|
||||
content=_remove_keyword_marker(result["Text"]),
|
||||
url=result["Url"],
|
||||
publishedDate=datetime.strptime(result["Published"], "%Y-%m-%d %H:%M:%S"),
|
||||
publishedDate=datetime.fromisoformat(result["Published"]),
|
||||
metadata=gettext.gettext("Posted by {author}").format(author=result["Author"]),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -43,7 +43,7 @@ def response(resp):
|
||||
|
||||
publishedDate = None
|
||||
if recipe['submissionDate']:
|
||||
publishedDate = datetime.strptime(result['recipe']['submissionDate'][:19], "%Y-%m-%dT%H:%M:%S")
|
||||
publishedDate = datetime.fromisoformat(result['recipe']['submissionDate'][:19])
|
||||
|
||||
content = [
|
||||
f"Schwierigkeitsstufe (1-3): {recipe['difficulty']}",
|
||||
|
||||
@@ -37,7 +37,7 @@ def response(resp):
|
||||
for item in search_res:
|
||||
img = 'https://s3.thehackerblog.com/findthatmeme/' + item['image_path']
|
||||
thumb = 'https://s3.thehackerblog.com/findthatmeme/thumb/' + item.get('thumbnail', '')
|
||||
date = datetime.strptime(item["updated_at"].split("T")[0], "%Y-%m-%d")
|
||||
date = datetime.fromisoformat(item["updated_at"].split("T")[0])
|
||||
formatted_date = datetime.fromtimestamp(date.timestamp())
|
||||
|
||||
results.append(
|
||||
|
||||
@@ -47,7 +47,7 @@ def response(resp: "SXNG_Response"):
|
||||
title=result["title"],
|
||||
content=result["description"],
|
||||
thumbnail=result["smallImageURL"],
|
||||
publishedDate=datetime.strptime(result["status_since"], "%Y-%m-%d %H:%M:%S"),
|
||||
publishedDate=datetime.fromisoformat(result["status_since"]),
|
||||
metadata=f"Rank: {result['rank']} || {result['episode_count']} episodes",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -68,7 +68,9 @@ def _get_api_token(query: str) -> str:
|
||||
if cached_token:
|
||||
return cached_token
|
||||
|
||||
resp = get(f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent()})
|
||||
resp = get(
|
||||
f"{base_url}/search?q={query}", headers={"User-Agent": gen_useragent(), "Accept-Language": "en-US,en:q=0.9"}
|
||||
)
|
||||
if not resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain request token: invalid response code")
|
||||
|
||||
@@ -76,7 +78,7 @@ def _get_api_token(query: str) -> str:
|
||||
if not token:
|
||||
raise SearxEngineAPIException("failed to obtain request token: no token found")
|
||||
|
||||
CACHE.set("token", token)
|
||||
CACHE.set("token", token, expire=3 * 60)
|
||||
return token
|
||||
|
||||
|
||||
|
||||
@@ -91,7 +91,7 @@ def response(resp) -> EngineResults:
|
||||
|
||||
published_date = None
|
||||
try:
|
||||
published_date = datetime.strptime(entry["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
published_date = datetime.fromisoformat(entry["createdAt"].rstrip("Z"))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ def _result(video: dict[str, typing.Any], album_info: dict[str, typing.Any]):
|
||||
release_time = album_info.get("releaseTime", {}).get("value")
|
||||
if release_time:
|
||||
try:
|
||||
published_date = datetime.strptime(release_time, "%Y-%m-%d")
|
||||
published_date = datetime.fromisoformat(release_time)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ def _get_communities(json):
|
||||
'title': result['community']['title'],
|
||||
'content': markdown_to_text(result['community'].get('description', '')),
|
||||
'thumbnail': result['community'].get('icon', result['community'].get('banner')),
|
||||
'publishedDate': datetime.strptime(counts['published'][:19], '%Y-%m-%dT%H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(counts['published'][:19]),
|
||||
'metadata': metadata,
|
||||
}
|
||||
)
|
||||
@@ -141,7 +141,7 @@ def _get_posts(json):
|
||||
'title': result['post']['name'],
|
||||
'content': content,
|
||||
'thumbnail': thumbnail,
|
||||
'publishedDate': datetime.strptime(result['post']['published'][:19], '%Y-%m-%dT%H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(result['post']['published'][:19]),
|
||||
'metadata': metadata,
|
||||
}
|
||||
)
|
||||
@@ -170,7 +170,7 @@ def _get_comments(json):
|
||||
'url': result['comment']['ap_id'],
|
||||
'title': result['post']['name'],
|
||||
'content': markdown_to_text(result['comment']['content']),
|
||||
'publishedDate': datetime.strptime(result['comment']['published'][:19], '%Y-%m-%dT%H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(result['comment']['published'][:19]),
|
||||
'metadata': metadata,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -60,7 +60,7 @@ def response(resp):
|
||||
'title': result['username'] + f" ({result['followers_count']} followers)",
|
||||
'content': result['note'],
|
||||
'thumbnail': result.get('avatar'),
|
||||
'publishedDate': datetime.strptime(result['created_at'][:10], "%Y-%m-%d"),
|
||||
'publishedDate': datetime.fromisoformat(result['created_at'][:10]),
|
||||
}
|
||||
)
|
||||
elif mastodon_type == "hashtags":
|
||||
|
||||
@@ -44,7 +44,7 @@ def response(resp) -> EngineResults:
|
||||
|
||||
cve_id = item["cve"]["id"]
|
||||
description = item["cve"]["descriptions"][0]["value"]
|
||||
date = datetime.strptime(item["cve"]["published"], "%Y-%m-%dT%H:%M:%S.%f")
|
||||
date = datetime.fromisoformat(item["cve"]["published"])
|
||||
|
||||
# Extract severity (Low, Medium, High, or Critical) and CVSS score, if available
|
||||
info = item["cve"].get("metrics", {}).get("cvssMetricV31", [{}])[0].get("cvssData", {})
|
||||
|
||||
@@ -76,7 +76,7 @@ def response(resp):
|
||||
release_time = item["release_time"]
|
||||
duration = item["duration"]
|
||||
|
||||
release_date = datetime.strptime(release_time.split("T")[0], "%Y-%m-%d")
|
||||
release_date = datetime.fromisoformat(release_time.split("T")[0])
|
||||
formatted_date = datetime.fromtimestamp(release_date.timestamp())
|
||||
|
||||
url = f"https://odysee.com/{name}:{claim_id}"
|
||||
|
||||
@@ -54,7 +54,7 @@ def response(resp: "SXNG_Response"):
|
||||
title=result["title"],
|
||||
content=result["description"],
|
||||
thumbnail=result["image_url"],
|
||||
publishedDate=datetime.strptime(result["created_at"], "%Y-%m-%d %H:%M:%S"),
|
||||
publishedDate=datetime.fromisoformat(result["created_at"]),
|
||||
metadata=" | ".join(metadata),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -140,7 +140,7 @@ def response(resp):
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'url': _clean_url(f"{about['website']}/images/{result['objectID']}"),
|
||||
'url': _clean_url(f"{pdia_base_url}/images/{result['objectID']}"),
|
||||
'img_src': _clean_url(base_image_url),
|
||||
'thumbnail_src': _clean_url(base_image_url + THUMBNAIL_SUFFIX),
|
||||
'title': f"{result['title'].strip()} by {result['artist']} {result.get('displayYear', '')}",
|
||||
|
||||
@@ -292,7 +292,7 @@ def parse_news_uchq(data):
|
||||
results = []
|
||||
for item in data.get('feed', []):
|
||||
try:
|
||||
published_date = datetime.strptime(item.get('time'), "%Y-%m-%d")
|
||||
published_date = datetime.fromisoformat(item.get('time'))
|
||||
except (ValueError, TypeError):
|
||||
# Sometime Quark will return non-standard format like "1天前", set published_date as None
|
||||
published_date = None
|
||||
|
||||
@@ -51,6 +51,7 @@ from urllib.parse import urlencode
|
||||
import babel
|
||||
from flask_babel import gettext # pyright: ignore[reportUnknownVariableType]
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
from searx.exceptions import (
|
||||
SearxEngineAccessDeniedException,
|
||||
@@ -105,9 +106,19 @@ qwant_news_locales = [
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
base_url = "https://www.qwant.com"
|
||||
api_url = "https://api.qwant.com/v3/search/"
|
||||
"""URL of Qwant's API (JSON)"""
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache for storing the ``datadome`` cookie."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]) -> bool:
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
return True
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
"""Qwant search request"""
|
||||
@@ -129,24 +140,29 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
"tgp": test_group_value,
|
||||
"device": "desktop",
|
||||
"safesearch": params["safesearch"],
|
||||
"display": True,
|
||||
"llm": True,
|
||||
# True would be encoded to "True", instead of "true", which makes the request
|
||||
# easier to detect and block
|
||||
"displayed": "true",
|
||||
"llm": "true",
|
||||
}
|
||||
|
||||
# shuffle query parameters to be harder to fingerprint
|
||||
args = list(args.items())
|
||||
random.shuffle(args)
|
||||
args = dict(args)
|
||||
|
||||
params["raise_for_httperror"] = False
|
||||
|
||||
params["url"] = f"{api_url}{qwant_categ}?{urlencode(args)}"
|
||||
|
||||
params["cookies"]["datadome"] = CACHE.get("datadome")
|
||||
params["headers"].update({"Accept": "application/json", "Referer": f"{base_url}/", "Origin": base_url})
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
"""Parse results from Qwant's API"""
|
||||
# pylint: disable=too-many-locals, too-many-branches, too-many-statements
|
||||
|
||||
# cache datadome cookie - changes on each request
|
||||
datadome = resp.cookies.get("datadome")
|
||||
if datadome:
|
||||
CACHE.set("datadome", datadome)
|
||||
|
||||
res = EngineResults()
|
||||
|
||||
# Try to load JSON result
|
||||
@@ -163,8 +179,8 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
error_code = data.get("error_code")
|
||||
if error_code == 24:
|
||||
raise SearxEngineTooManyRequestsException()
|
||||
if search_results.get("data", {}).get("error_data", {}).get("captchaUrl") is not None:
|
||||
raise SearxEngineCaptchaException()
|
||||
if search_results.get("url") is not None:
|
||||
raise SearxEngineCaptchaException(suspended_time=0)
|
||||
if resp.status_code == 403:
|
||||
raise SearxEngineAccessDeniedException()
|
||||
msg = ",".join(data.get("message", ["unknown"]))
|
||||
@@ -302,7 +318,7 @@ def fetch_traits(engine_traits: EngineTraits):
|
||||
from searx.utils import extr
|
||||
|
||||
resp = get(
|
||||
about["website"], # pyright: ignore[reportArgumentType]
|
||||
base_url, # pyright: ignore[reportArgumentType]
|
||||
timeout=5,
|
||||
)
|
||||
if not resp.ok:
|
||||
|
||||
@@ -58,7 +58,7 @@ def response(resp):
|
||||
title = extract_text(result_dom.xpath(title_xpath))
|
||||
p_date = extract_text(result_dom.xpath(published_date))
|
||||
# fix offset date for line 644 webapp.py check
|
||||
fixed_date = datetime.strptime(p_date, '%Y-%m-%dT%H:%M:%S%z')
|
||||
fixed_date = datetime.fromisoformat(p_date)
|
||||
earned = extract_text(result_dom.xpath(earned_xpath))
|
||||
views = extract_text(result_dom.xpath(views_xpath))
|
||||
rumbles = extract_text(result_dom.xpath(rumbles_xpath))
|
||||
|
||||
@@ -120,7 +120,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
|
||||
publishedDate: datetime | None
|
||||
if "pubDate" in result:
|
||||
publishedDate = datetime.strptime(result["pubDate"], "%Y-%m-%d")
|
||||
publishedDate = datetime.fromisoformat(result["pubDate"])
|
||||
else:
|
||||
publishedDate = None
|
||||
|
||||
|
||||
@@ -95,7 +95,8 @@ def _parse_date(text):
|
||||
date_match = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", text)
|
||||
if date_match:
|
||||
try:
|
||||
return datetime.strptime(date_match.group(1), "%Y-%m-%d")
|
||||
y, m, d = date_match.group(1).split("-")
|
||||
return datetime(year=int(y), month=int(m), day=int(d))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
@@ -54,7 +54,7 @@ def response(resp):
|
||||
published_date = None
|
||||
if entry.get("date") and entry.get("duration"):
|
||||
try:
|
||||
published_date = datetime.strptime(entry['date'], "%Y-%m-%d")
|
||||
published_date = datetime.fromisoformat(entry['date'])
|
||||
except (ValueError, TypeError):
|
||||
published_date = None
|
||||
|
||||
|
||||
@@ -134,7 +134,7 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
return str(record.get(k, ""))
|
||||
|
||||
for record in json_data["records"]:
|
||||
published = datetime.strptime(record["publicationDate"], "%Y-%m-%d")
|
||||
published = datetime.fromisoformat(record["publicationDate"])
|
||||
authors: list[str] = [" ".join(author["creator"].split(", ")[::-1]) for author in record["creators"]]
|
||||
|
||||
pdf_url = ""
|
||||
|
||||
@@ -81,7 +81,7 @@ def _story(item):
|
||||
return {
|
||||
'title': item['title'],
|
||||
'thumbnail': item.get('teaserImage', {}).get('imageVariants', {}).get('16x9-256'),
|
||||
'publishedDate': datetime.strptime(item['date'][:19], '%Y-%m-%dT%H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(item['date'][:19]),
|
||||
'content': item.get('firstSentence'),
|
||||
'url': item['shareURL'] if use_source_url else item['detailsweb'],
|
||||
}
|
||||
@@ -103,7 +103,7 @@ def _video(item):
|
||||
'template': 'videos.html',
|
||||
'title': title,
|
||||
'thumbnail': item.get('teaserImage', {}).get('imageVariants', {}).get('16x9-256'),
|
||||
'publishedDate': datetime.strptime(item['date'][:19], '%Y-%m-%dT%H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(item['date'][:19]),
|
||||
'content': item.get('firstSentence', ''),
|
||||
'iframe_src': video_url,
|
||||
'url': url,
|
||||
|
||||
@@ -117,7 +117,7 @@ def _obtain_session_code() -> str:
|
||||
if not code:
|
||||
raise SearxEngineAPIException("failed to obtain session code")
|
||||
|
||||
CACHE.set("session", code, expire=60 * 24 * 60) # cookie is valid for two months
|
||||
CACHE.set("session", code, expire=60 * 24 * 60 * 60) # cookie is valid for two months
|
||||
return code
|
||||
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ def parse_tineye_match(match_json):
|
||||
|
||||
crawl_date = backlink_json.get("crawl_date")
|
||||
if crawl_date:
|
||||
crawl_date = datetime.strptime(crawl_date, '%Y-%m-%d')
|
||||
crawl_date = datetime.fromisoformat(crawl_date)
|
||||
else:
|
||||
crawl_date = datetime.min
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ def response(resp):
|
||||
'title': title,
|
||||
'content': html_to_text(result['content']),
|
||||
'thumbnail': thumbnail,
|
||||
'publishedDate': datetime.strptime(result['created_at'], '%Y-%m-%d %H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(result['created_at']),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from dateutil import parser
|
||||
|
||||
from searx.exceptions import SearxEngineAPIException
|
||||
from searx.network import get
|
||||
from searx.utils import html_to_text
|
||||
from searx.utils import gen_useragent, html_to_text
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
@@ -52,7 +52,7 @@ def _obtain_x_sid() -> tuple[str, str]:
|
||||
The header key is usually called `x-sid-{UUIDv4}`, and the value is
|
||||
usually a plain UUIDv4 (but a different one than in the header key).
|
||||
"""
|
||||
resp = get(f"{api_url}/revcontent/embed.js")
|
||||
resp = get(f"{api_url}/revcontent/embed.js", headers={"User-Agent": gen_useragent()})
|
||||
if not resp.ok:
|
||||
raise SearxEngineAPIException("failed to obtain request x-sid token")
|
||||
|
||||
@@ -89,12 +89,14 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
params["url"] = f"{api_url}/Search/Web?{urlencode(args)}"
|
||||
|
||||
x_sid_header, x_sid_value = _obtain_x_sid()
|
||||
params["headers"] = {
|
||||
x_sid_header: x_sid_value,
|
||||
# required - we send a random longitude and latitude instead of the actual user location
|
||||
'x-lon': str(random.random() * 90),
|
||||
'x-lat': str(random.random() * 90),
|
||||
}
|
||||
params["headers"].update(
|
||||
{
|
||||
x_sid_header: x_sid_value,
|
||||
# required - we send a random longitude and latitude instead of the actual user location
|
||||
"x-lon": str(round(random.random() * 90, 4)),
|
||||
"x-lat": str(round(random.random() * 90, 4)),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
|
||||
@@ -79,7 +79,7 @@ def response(resp):
|
||||
'img_src': result['path'],
|
||||
'thumbnail_src': result['thumbs']['small'],
|
||||
'resolution': result['resolution'].replace('x', ' x '),
|
||||
'publishedDate': datetime.strptime(result['created_at'], '%Y-%m-%d %H:%M:%S'),
|
||||
'publishedDate': datetime.fromisoformat(result['created_at']),
|
||||
'img_format': result['file_type'],
|
||||
'filesize': humanize_bytes(result['file_size']),
|
||||
}
|
||||
|
||||
@@ -52,6 +52,12 @@ def _normalize_url_fields(result: "Result | LegacyResult"):
|
||||
result.parsed_url = urllib.parse.urlparse(result.url)
|
||||
|
||||
if result.parsed_url:
|
||||
# properly format special characters (e.g. "ä", "ö") in IDN domains
|
||||
# e.g. "xn--strung-xxa.de" becomes "störung.de"
|
||||
if result.parsed_url.netloc.startswith("xn--"):
|
||||
netloc = result.parsed_url.netloc.encode().decode("idna")
|
||||
result.parsed_url = result.parsed_url._replace(netloc=netloc)
|
||||
|
||||
result.parsed_url = result.parsed_url._replace(
|
||||
# if the result has no scheme, use http as default
|
||||
scheme=result.parsed_url.scheme or "http",
|
||||
|
||||
@@ -477,6 +477,24 @@ engines:
|
||||
engine: arxiv
|
||||
shortcut: arx
|
||||
|
||||
- name: avalw
|
||||
engine: json_engine
|
||||
shortcut: av
|
||||
categories: general
|
||||
paging: true
|
||||
search_url: https://avalw.org/api/search?q={query}&page={pageno}&limit=10&tbm=all
|
||||
results_query: results
|
||||
url_query: url
|
||||
title_query: title
|
||||
content_query: snippet
|
||||
thumbnail: image_url
|
||||
disabled: true
|
||||
inactive: true
|
||||
about:
|
||||
website: https://avalw.com
|
||||
description: "Search engine from the Romanian news company AVALW S.R.L."
|
||||
results: JSON
|
||||
|
||||
- name: ayo
|
||||
engine: xpath
|
||||
categories: general
|
||||
|
||||
@@ -60,7 +60,7 @@
|
||||
</p>
|
||||
<p class="result-filesize">{{ _label(_("Filesize"), result.filesize) }}</p>
|
||||
<p class="result-source">{{ _label(_("Source"), result.source) }}</p>
|
||||
<p class="result-engine">{{ _label(_("Engine"), result.engine) }}</p>
|
||||
<p class="result-engines">{{ _label(_("Engines"), ", ".join(result.engines)) }}</p>
|
||||
<p class="result-url"><span>{{ _("View source") }}:</span>{{- "" -}}
|
||||
<a {{ _target(results_on_new_tab) }} href="{{ result.url }}">{{ result.url }}</a>
|
||||
</p>
|
||||
|
||||
@@ -845,7 +845,10 @@ def autocompleter():
|
||||
mimetype = 'application/json'
|
||||
else:
|
||||
# the suggestion request comes from browser's URL bar
|
||||
suggestions = json.dumps([sug_prefix, results])
|
||||
relevances = {
|
||||
'google:suggestrelevance': [600 - i for i in range(len(results))]
|
||||
} # chromium only shows 3 suggestions unless we attach relevances
|
||||
suggestions = json.dumps([sug_prefix, results, [], [], relevances])
|
||||
mimetype = 'application/x-suggestions+json'
|
||||
|
||||
suggestions = escape(suggestions, False)
|
||||
|
||||
Reference in New Issue
Block a user