mirror of
https://github.com/searxng/searxng.git
synced 2026-09-11 16:56:05 +00:00
Compare commits
7 Commits
3fdc6d753a
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
61d660276f | ||
|
6a27c21008 | ||
|
ffe96f8a6f | ||
|
|
931fd9787b | ||
|
|
42e1d61296 | ||
|
|
765a9999df | ||
|
ba055b3e09 |
88
client/simple/package-lock.json
generated
88
client/simple/package-lock.json
generated
@@ -15,8 +15,8 @@
|
||||
"swiped-events": "1.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.5.11",
|
||||
"@types/node": "^26.4.0",
|
||||
"@biomejs/biome": "2.5.12",
|
||||
"@types/node": "^26.5.0",
|
||||
"browserslist": "^4.28.8",
|
||||
"browserslist-to-esbuild": "^2.1.1",
|
||||
"edge.js": "^6.5.1",
|
||||
@@ -69,9 +69,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/biome": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.11.tgz",
|
||||
"integrity": "sha512-Tj0dnkLPdW0ASjHfj2D/ZkkvPU2wrFmnE1jWTD2xzV1ycapV1DutbYXk4NDnR3rYTi1ZCbNFD4G2gRMEY65WaA==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.5.12.tgz",
|
||||
"integrity": "sha512-Lw4VHZRebrReBBnlHa12JQjnIBm3JJAA55PDB9LbBVBF0q4RYphm6KfmIjqtPhf61MxZ5Q9KoK8R8x+7per5Aw==",
|
||||
"dev": true,
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"bin": {
|
||||
@@ -85,20 +85,20 @@
|
||||
"url": "https://opencollective.com/biome"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@biomejs/cli-darwin-arm64": "2.5.11",
|
||||
"@biomejs/cli-darwin-x64": "2.5.11",
|
||||
"@biomejs/cli-linux-arm64": "2.5.11",
|
||||
"@biomejs/cli-linux-arm64-musl": "2.5.11",
|
||||
"@biomejs/cli-linux-x64": "2.5.11",
|
||||
"@biomejs/cli-linux-x64-musl": "2.5.11",
|
||||
"@biomejs/cli-win32-arm64": "2.5.11",
|
||||
"@biomejs/cli-win32-x64": "2.5.11"
|
||||
"@biomejs/cli-darwin-arm64": "2.5.12",
|
||||
"@biomejs/cli-darwin-x64": "2.5.12",
|
||||
"@biomejs/cli-linux-arm64": "2.5.12",
|
||||
"@biomejs/cli-linux-arm64-musl": "2.5.12",
|
||||
"@biomejs/cli-linux-x64": "2.5.12",
|
||||
"@biomejs/cli-linux-x64-musl": "2.5.12",
|
||||
"@biomejs/cli-win32-arm64": "2.5.12",
|
||||
"@biomejs/cli-win32-x64": "2.5.12"
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-darwin-arm64": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.11.tgz",
|
||||
"integrity": "sha512-6SGZxoKbXvUjMn1t6A98HqWISPnGNbYs0R/Rt2JarmXBSev+lva4QxUMWEBX9lX1Wo1XTJ78uk5xVDtG58SRZg==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.5.12.tgz",
|
||||
"integrity": "sha512-lCRY1rwgNeWNgTr4DI/u6ZwXTRwRLHAvbaio1YLLGS+4r1nhvB2ssyPqIpfUSmRveNfv0fn/N58C7CAdK2XVrg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -113,9 +113,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-darwin-x64": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.11.tgz",
|
||||
"integrity": "sha512-nYkXY7tLBEgnGbYapDKAyKzgt44ZEyG+AKalvTXtCWKYgepI9dw327q+cVgedxm+Udi1ZzHKUyZrIusHi/KQbw==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.5.12.tgz",
|
||||
"integrity": "sha512-vhPgwnh+6tN3ArdAXuET99xaNbFt7CG82Bqn+omHVLC5xdVx45JsYjGPmUIGNzjDek5XdNCP1HKksK7fn8+3bQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -130,9 +130,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-linux-arm64": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.11.tgz",
|
||||
"integrity": "sha512-3PVLSTD9RR73rvVPt5G3T1gc+ycggWEGfTD7RvzzbtcDPD27NxgxBbAFfpm7DXJKW6VLHWE1lLMGvFt2Qxjcow==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.5.12.tgz",
|
||||
"integrity": "sha512-2gp8aVwXYKdAtmBfRFCUuyDMcfN1ahHqUkGfLYrZlNRFmryMATLVvJgWKvyA8wu4Rwn5OSxM1UcUmOuOFNGeBQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -150,9 +150,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-linux-arm64-musl": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.11.tgz",
|
||||
"integrity": "sha512-qhyZUMyCbWYFV2bAwRNVvfMVZ+hv7WYl6mossGrxC+uiQQXhvsuWWU8zz6jYX0mChZd9MgQZbm4vozTmG/5iGw==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.5.12.tgz",
|
||||
"integrity": "sha512-couHYjFLL5uuI8ne6zhT7KwEsXo5YP7ry/2xmEqah7qanu0YmfDi3mwJg47YXSuv/NpZj22CZzcRH/5c4gjPSQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -170,9 +170,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-linux-x64": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.11.tgz",
|
||||
"integrity": "sha512-JOytptlsgM33B2MMFUg8iBrb4IKpbD5JnJrSeYiaFEeAj4vuXx0iQSQZ4qK7sqyMtfjZxxPdNdMZZVL4y/mFyA==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.5.12.tgz",
|
||||
"integrity": "sha512-SnvOs3TSTiuia4SQOUNe1aWC9RT4+YkjcKnOhL/nsKOV0k5ycgBkDzF0lUxKn1V7Q8CLTRq6iV23ZAivHomRoA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -190,9 +190,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-linux-x64-musl": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.11.tgz",
|
||||
"integrity": "sha512-oRRlrchG5EfrEL/EmtT1qUjSNHk3/5LGeZhQqADBBAJF1b1ET6964xEKe7aGlGARzDfza8H/seEsFJl7S6Ql9w==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.5.12.tgz",
|
||||
"integrity": "sha512-8A0oDW58/w9f/PQNYuq0sGUZtGtGrkNF4Z6n0PUoXpLCshi85vtKTv1XSznQawhdE4MXJ8ufpzHXyLFe87M/+w==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -210,9 +210,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-win32-arm64": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.11.tgz",
|
||||
"integrity": "sha512-e49E6K9hzH/ohJNx8Y26mY8HaV4I4ZViIeoqhKsmoXLKHhQnMeBAVqCgsGf2Wa3lXlS7RkporDXMHHWkzvZzFw==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.5.12.tgz",
|
||||
"integrity": "sha512-b9vtoZFsuZt1pdjNwJvXl0f+BpayRzV008uS2+JpmwIKdSE2qdu4A/l04FESwLoou5g2E/Qlec0xwJydZplH+A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -227,9 +227,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@biomejs/cli-win32-x64": {
|
||||
"version": "2.5.11",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.11.tgz",
|
||||
"integrity": "sha512-QSQr/KjOgXA7OzXJUWS+oguKyAZ3Q0l/lnlDGbu397eKo83atuWUjBPJrsqbKNF6CARGw8XXJLGzpHC8Ryhd4Q==",
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.5.12.tgz",
|
||||
"integrity": "sha512-B1R/l+CwEpKFSuqiwePzPNRk1EiJN8kc0UhdafNz6MZN9v5OFP9HYP1irptvWzHrwVI4blVNGMbxc5zt70m3IA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1601,13 +1601,13 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "26.4.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz",
|
||||
"integrity": "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA==",
|
||||
"version": "26.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.0.tgz",
|
||||
"integrity": "sha512-dVSGpriSoCgz8WnDNTuSSuSv1PC/ALXihO4ulRZt7Md8k9mlbdin3lGOcDE8SnWOgf513ByWlXd7BK4azmyg/A==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~8.3.0"
|
||||
"undici-types": "~8.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/pluralize": {
|
||||
@@ -5004,9 +5004,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "8.3.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
|
||||
"integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
|
||||
"version": "8.9.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
|
||||
"integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
|
||||
@@ -29,8 +29,8 @@
|
||||
"swiped-events": "1.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.5.11",
|
||||
"@types/node": "^26.4.0",
|
||||
"@biomejs/biome": "2.5.12",
|
||||
"@types/node": "^26.5.0",
|
||||
"browserslist": "^4.28.8",
|
||||
"browserslist-to-esbuild": "^2.1.1",
|
||||
"edge.js": "^6.5.1",
|
||||
|
||||
8
docs/dev/engines/online/europepmc.rst
Normal file
8
docs/dev/engines/online/europepmc.rst
Normal file
@@ -0,0 +1,8 @@
|
||||
.. _europepmc engine:
|
||||
|
||||
==========
|
||||
Europe PMC
|
||||
==========
|
||||
|
||||
.. automodule:: searx.engines.europepmc
|
||||
:members:
|
||||
@@ -72,43 +72,21 @@ def get_locale_params(engine_region: str | None) -> dict[str, str] | None:
|
||||
return {"mkt": engine_region}
|
||||
|
||||
|
||||
def override_accept_language(params: "OnlineParams", engine_region: str | None) -> None:
|
||||
"""Override the ``Accept-Language`` header.
|
||||
|
||||
The default header built by :py:class:`~searx.search.processors.online.OnlineProcessor`
|
||||
appends ``en;q=0.3`` as a fallback language::
|
||||
|
||||
Accept-Language: de,de-DE;q=0.7,en;q=0.3
|
||||
|
||||
Bing seems to better select the results locale based on the
|
||||
``Accept-Language`` value header.
|
||||
|
||||
This function is shared with :py:mod:`searx.engines.bing_images`,
|
||||
:py:mod:`searx.engines.bing_news`, and :py:mod:`searx.engines.bing_videos`.
|
||||
"""
|
||||
|
||||
if not engine_region or engine_region == "clear":
|
||||
return
|
||||
|
||||
lang = engine_region.split("-")[0]
|
||||
params["headers"]["Accept-Language"] = f"{engine_region},{lang};q=0.9"
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams"):
|
||||
"""Assemble a Bing-Web request."""
|
||||
|
||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||
|
||||
override_accept_language(params, engine_region)
|
||||
|
||||
query_params: dict[str, str | int] = {
|
||||
"q": query,
|
||||
"adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
|
||||
}
|
||||
|
||||
locale_params = get_locale_params(engine_region)
|
||||
if locale_params:
|
||||
query_params.update(locale_params)
|
||||
if engine_region and engine_region != "clear":
|
||||
lang, _, cc = engine_region.partition("-")
|
||||
query_params["setlang"] = lang
|
||||
if cc and cc not in ("us", "cn", "ru"): # bing just sends junk for these
|
||||
query_params["cc"] = cc
|
||||
|
||||
params["url"] = f"{base_url}/search?{urlencode(query_params)}"
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ from lxml import html
|
||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||
fetch_traits,
|
||||
get_locale_params,
|
||||
override_accept_language,
|
||||
)
|
||||
|
||||
# about
|
||||
@@ -44,8 +43,6 @@ def request(query, params):
|
||||
|
||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||
|
||||
override_accept_language(params, engine_region)
|
||||
|
||||
# build URL query
|
||||
# - example: https://www.bing.com/images/async?q=foo&async=1&first=1&count=35
|
||||
query_params = {
|
||||
|
||||
@@ -12,10 +12,7 @@ from urllib.parse import urlencode
|
||||
from lxml import html
|
||||
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
from searx.engines.bing import (
|
||||
get_locale_params,
|
||||
override_accept_language,
|
||||
)
|
||||
from searx.engines.bing import get_locale_params
|
||||
from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
|
||||
|
||||
# about
|
||||
@@ -54,8 +51,6 @@ def request(query, params):
|
||||
|
||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||
|
||||
override_accept_language(params, engine_region)
|
||||
|
||||
# build URL query
|
||||
# - example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
|
||||
page = int(params.get("pageno", 1)) - 1
|
||||
|
||||
@@ -9,7 +9,6 @@ from lxml import html
|
||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||
fetch_traits,
|
||||
get_locale_params,
|
||||
override_accept_language,
|
||||
)
|
||||
from searx.engines.bing_images import time_map
|
||||
from searx.utils import eval_xpath, eval_xpath_getindex
|
||||
@@ -39,8 +38,6 @@ def request(query, params):
|
||||
|
||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||
|
||||
override_accept_language(params, engine_region)
|
||||
|
||||
# build URL query
|
||||
# - example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
|
||||
query_params = {
|
||||
|
||||
@@ -141,12 +141,13 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
||||
if name:
|
||||
authors.add(name)
|
||||
|
||||
tag = result.get("fieldOfStudy")
|
||||
res.add(
|
||||
res.types.Paper(
|
||||
title=result.get("title"),
|
||||
url=url,
|
||||
content=result.get("fullText", "") or "",
|
||||
tags=result.get("fieldOfStudy", []),
|
||||
tags=[tag] if tag else [],
|
||||
publishedDate=published_date,
|
||||
type=result.get("documentType", "") or "",
|
||||
authors=authors,
|
||||
|
||||
@@ -14,11 +14,12 @@ can't build it ourselves and must scrape it from the HTML pages.
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
import re
|
||||
|
||||
from urllib.parse import quote_plus
|
||||
from urllib.parse import quote_plus, urljoin
|
||||
from lxml import html
|
||||
|
||||
from searx.utils import html_to_text, gen_useragent, extract_text, eval_xpath
|
||||
from searx.utils import html_to_text, extract_text, eval_xpath
|
||||
from searx.result_types import EngineResults
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.network import get
|
||||
@@ -38,7 +39,6 @@ about = {
|
||||
# engine dependent config
|
||||
categories = ["general"]
|
||||
paging = True
|
||||
_HTTP_User_Agent: str = gen_useragent()
|
||||
|
||||
base_url = "https://duckduckgo.com"
|
||||
|
||||
@@ -73,6 +73,8 @@ def _fetch_first_page_link(
|
||||
resp = get(
|
||||
url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
|
||||
headers=headers,
|
||||
impersonate="firefox",
|
||||
default_headers=False,
|
||||
timeout=2,
|
||||
)
|
||||
|
||||
@@ -96,6 +98,43 @@ def _cache_key(query: str, pageno: int) -> str:
|
||||
return f"nextpage_url|{query}|{pageno}"
|
||||
|
||||
|
||||
def _solve_jsa(resp: "SXNG_Response") -> "SXNG_Response":
|
||||
"""Duckduckgo sometimes issues a challenge instead of json."""
|
||||
|
||||
# length that a real browser would report for where the broken snippet is
|
||||
html_len = {
|
||||
"<p><div></p><p></div": 32,
|
||||
"<li><div></li><li></div": 29,
|
||||
"<div><div></div><div></div": 33,
|
||||
"<br><div></br><br></div": 23,
|
||||
}
|
||||
|
||||
js = resp.text or ""
|
||||
jsa_match = re.search(r"let jsa = (\d+);.*?DDG\.deep\.initialize\('([^']+)'", js, re.S)
|
||||
if not jsa_match:
|
||||
return resp
|
||||
|
||||
js_functions = dict(re.findall(r"let (\w+) = function\(num\) \{([^}]*)\};", js))
|
||||
jsa = int(jsa_match.group(1))
|
||||
try:
|
||||
for name in re.findall(r"jsa = (\w+)\(jsa\);", js):
|
||||
body = js_functions[name]
|
||||
mul = re.search(r"num \* (\d+)", body)
|
||||
jsa = jsa * int(mul.group(1)) if mul else jsa + html_len[re.search(r"`([^`]+)`", body).group(1)]
|
||||
except (KeyError, AttributeError):
|
||||
return resp
|
||||
|
||||
params = resp.search_params
|
||||
follow = get(
|
||||
urljoin("https://links.duckduckgo.com", jsa_match.group(2) + str(jsa)),
|
||||
headers=params["headers"],
|
||||
impersonate="firefox",
|
||||
default_headers=False,
|
||||
)
|
||||
follow.search_params = params
|
||||
return follow
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
if len(query) >= 500:
|
||||
@@ -103,25 +142,15 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
params["url"] = None
|
||||
return
|
||||
|
||||
headers = params["headers"]
|
||||
|
||||
# The vqd value is generated from the query and the UA header. To be able
|
||||
# to reuse the vqd value, the UA header must be static.
|
||||
headers["User-Agent"] = _HTTP_User_Agent
|
||||
headers["Accept"] = "*/*"
|
||||
headers["Referer"] = f"{base_url}/"
|
||||
headers["Host"] = "duckduckgo.com"
|
||||
|
||||
# Sec-Fetch headers are required to not get blocked when sending a Firefox user agent
|
||||
headers["Sec-Fetch-Dest"] = "script"
|
||||
headers["Sec-Fetch-Mode"] = "no-cors"
|
||||
headers["Sec-Fetch-Site"] = "same-site"
|
||||
# firefox TLS only
|
||||
params["impersonate"] = "firefox"
|
||||
params["default_headers"] = False
|
||||
|
||||
api_url = ""
|
||||
if params["pageno"] > 1:
|
||||
api_url = CACHE.get(_cache_key(query, params["pageno"]))
|
||||
else:
|
||||
api_url = _fetch_first_page_link(query, headers)
|
||||
api_url = _fetch_first_page_link(query, params["headers"])
|
||||
|
||||
if not api_url:
|
||||
params["url"] = None
|
||||
@@ -129,14 +158,27 @@ def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
|
||||
|
||||
# loads as a script
|
||||
headers = params["headers"]
|
||||
headers["Accept"] = "*/*"
|
||||
headers["Sec-Fetch-Dest"] = "script"
|
||||
headers["Sec-Fetch-Mode"] = "no-cors"
|
||||
headers["Sec-Fetch-Site"] = "same-site"
|
||||
headers["Referer"] = f"{base_url}/"
|
||||
|
||||
# TODO: support safesearch, timerange and engine traits # pylint:disable=fixme
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response"):
|
||||
res = EngineResults()
|
||||
res_json = resp.json()
|
||||
|
||||
for result in res_json["results"]:
|
||||
# check if ddg returns a challenge
|
||||
# e.g. 'site:github.com searxng'
|
||||
if "let jsa =" in (resp.text or ""):
|
||||
resp = _solve_jsa(resp)
|
||||
|
||||
results = resp.json()["results"]
|
||||
for result in results:
|
||||
if "u" not in result:
|
||||
continue
|
||||
|
||||
@@ -144,13 +186,13 @@ def response(resp: "SXNG_Response"):
|
||||
res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
|
||||
)
|
||||
|
||||
# link to next page
|
||||
next_page_path = res_json["results"][-1].get("n")
|
||||
if next_page_path:
|
||||
CACHE.set(
|
||||
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
|
||||
base_url + next_page_path,
|
||||
expire=60 * 60,
|
||||
)
|
||||
if results:
|
||||
next_page_path = results[-1].get("n")
|
||||
if next_page_path:
|
||||
CACHE.set(
|
||||
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
|
||||
base_url + next_page_path,
|
||||
expire=60 * 60,
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
150
searx/engines/europepmc.py
Normal file
150
searx/engines/europepmc.py
Normal file
@@ -0,0 +1,150 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""`Europe PMC`_ provides comprehensive access to life sciences literature from
|
||||
trusted sources. With Europe PMC you can search and read millions of
|
||||
publications, preprints and other documents enriched with links to supporting
|
||||
data, reviews, protocols, and other relevant resources.
|
||||
|
||||
.. _Europe PMC: https://europepmc.org/
|
||||
|
||||
Configuration
|
||||
=============
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
- name: europepmc
|
||||
engine: europepmc
|
||||
shortcut: epmc
|
||||
|
||||
Implementations
|
||||
===============
|
||||
|
||||
"""
|
||||
|
||||
import typing as t
|
||||
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from dateutil.parser import isoparse
|
||||
|
||||
from searx.enginelib import EngineCache
|
||||
from searx.result_types import EngineResults
|
||||
from searx.utils import html_to_text
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
|
||||
about = {
|
||||
"website": "https://europepmc.org/",
|
||||
"wikidata_id": "Q5412157",
|
||||
"official_api_documentation": "https://europepmc.org/RestfulWebService",
|
||||
"use_official_api": True,
|
||||
"require_api_key": False,
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
categories = ["science", "scientific publications"]
|
||||
paging = True
|
||||
|
||||
# engine dependent config
|
||||
search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
|
||||
article_url = "https://europepmc.org/article/"
|
||||
|
||||
page_size = 20
|
||||
|
||||
CACHE: EngineCache
|
||||
"""Cache for storing the pagination cursor."""
|
||||
|
||||
|
||||
def setup(engine_settings: dict[str, t.Any]):
|
||||
global CACHE # pylint: disable=global-statement
|
||||
CACHE = EngineCache(engine_settings["name"])
|
||||
|
||||
|
||||
def _cache_key(query: str, page: int) -> str:
|
||||
return f"{query}|{page}"
|
||||
|
||||
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
args = {
|
||||
"query": query,
|
||||
"format": "json",
|
||||
"resultType": "core",
|
||||
"pageSize": page_size,
|
||||
}
|
||||
|
||||
if params["pageno"] > 1:
|
||||
if cursor := CACHE.get(_cache_key(query, params["pageno"])):
|
||||
args["cursorMark"] = cursor
|
||||
else:
|
||||
# no cached cursor for that page
|
||||
params["url"] = None
|
||||
return
|
||||
|
||||
params["url"] = f"{search_url}?{urlencode(args)}"
|
||||
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res = EngineResults()
|
||||
|
||||
json_resp = resp.json()
|
||||
|
||||
# store pagination cursor for loading next pages in cache
|
||||
if next_cursor := json_resp.get("nextCursorMark"):
|
||||
next_page = resp.search_params["pageno"] + 1
|
||||
query = resp.search_params["query"]
|
||||
CACHE.set(_cache_key(query, next_page), next_cursor)
|
||||
|
||||
all_results = json_resp.get("resultList", {}).get("result", [])
|
||||
|
||||
for item in all_results:
|
||||
source = item.get("source", "")
|
||||
identifier = item.get("id", "")
|
||||
url = f"{article_url}{source}/{identifier}" if source and identifier else ""
|
||||
|
||||
journal_info: dict[str, t.Any] = item.get("journalInfo", {})
|
||||
journal: dict[str, t.Any] = journal_info.get("journal", {})
|
||||
|
||||
res.add(
|
||||
res.types.Paper(
|
||||
url=url,
|
||||
title=html_to_text(item.get("title", "")),
|
||||
content=html_to_text(item.get("abstractText", "")),
|
||||
journal=journal.get("title", ""),
|
||||
issn=[journal.get("issn", "")],
|
||||
authors=_get_authors(item),
|
||||
doi=item.get("doi", ""),
|
||||
publishedDate=_get_published_date(item),
|
||||
type=", ".join((item.get("pubTypeList", {})).get("pubType", [])),
|
||||
pdf_url=_get_pdf_url(item),
|
||||
html_url=url,
|
||||
)
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
def _get_authors(item: dict[str, t.Any]) -> list:
|
||||
"""Extract the list of authors from the item."""
|
||||
if authors := item.get("authorString", None):
|
||||
authors = [author.strip().rstrip(".") for author in authors.split(",") if author.strip()]
|
||||
else:
|
||||
authors = []
|
||||
return authors
|
||||
|
||||
|
||||
def _get_pdf_url(item: dict[str, t.Any]) -> str:
|
||||
"""Extract the PDF URL in case it is open access."""
|
||||
for url_info in (item.get("fullTextUrlList", {})).get("fullTextUrl", []):
|
||||
if url_info.get("documentStyle") == "pdf" and url_info.get("availabilityCode") == "OA":
|
||||
return url_info.get("url", "")
|
||||
return ""
|
||||
|
||||
|
||||
def _get_published_date(item: dict[str, t.Any]) -> datetime | None:
|
||||
"""Extract the published date from the item and convert it to a datetime object."""
|
||||
if unformatted_date := item.get("firstPublicationDate"):
|
||||
return isoparse(unformatted_date)
|
||||
return None
|
||||
@@ -2,72 +2,85 @@
|
||||
"""Pinterest (images)"""
|
||||
|
||||
from json import dumps
|
||||
import typing as t
|
||||
from searx.result_types import EngineResults
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from searx.extended_types import SXNG_Response
|
||||
from searx.search.processors import OnlineParams
|
||||
|
||||
about = {
|
||||
"website": 'https://www.pinterest.com/',
|
||||
"wikidata_id": 'Q255381',
|
||||
"official_api_documentation": 'https://developers.pinterest.com/docs/api/v5/',
|
||||
"website": "https://www.pinterest.com/",
|
||||
"wikidata_id": "Q255381",
|
||||
"official_api_documentation": "https://developers.pinterest.com/docs/api/v5/",
|
||||
"use_official_api": False,
|
||||
"require_api_key": False,
|
||||
"results": 'JSON',
|
||||
"results": "JSON",
|
||||
}
|
||||
|
||||
categories = ['images']
|
||||
categories = ["images"]
|
||||
paging = True
|
||||
|
||||
base_url = 'https://www.pinterest.com'
|
||||
base_url = "https://www.pinterest.com"
|
||||
|
||||
|
||||
def request(query, params):
|
||||
def request(query: str, params: "OnlineParams") -> None:
|
||||
|
||||
args = {
|
||||
'options': {
|
||||
'query': query,
|
||||
'bookmarks': [params['engine_data'].get('bookmark', '')],
|
||||
"options": {
|
||||
"query": query,
|
||||
"bookmarks": [params["engine_data"].get("bookmark", "")],
|
||||
},
|
||||
'context': {},
|
||||
"context": {},
|
||||
}
|
||||
params['url'] = f"{base_url}/resource/BaseSearchResource/get/?data={dumps(args)}"
|
||||
params['headers'] = {
|
||||
'X-Pinterest-AppState': 'active',
|
||||
'X-Pinterest-Source-Url': '/ideas/',
|
||||
'X-Pinterest-PWS-Handler': 'www/ideas.js',
|
||||
params["url"] = f"{base_url}/resource/BaseSearchResource/get/?data={dumps(args)}"
|
||||
params["headers"] = {
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"X-Pinterest-AppState": "active",
|
||||
"X-Pinterest-Source-Url": "/ideas/",
|
||||
"X-Pinterest-PWS-Handler": "www/ideas.js",
|
||||
}
|
||||
|
||||
return params
|
||||
|
||||
def response(resp: "SXNG_Response") -> EngineResults:
|
||||
res = EngineResults()
|
||||
json_resp: dict[str, t.Any] = resp.json() # type: ignore
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
|
||||
json_resp = resp.json()
|
||||
|
||||
results.append(
|
||||
res.add(
|
||||
{
|
||||
'engine_data': json_resp['resource_response']['bookmark'],
|
||||
"engine_data": json_resp["resource_response"]["bookmark"],
|
||||
# it's called bookmark by pinterest, but it's rather a nextpage
|
||||
# parameter to get the next results
|
||||
'key': 'bookmark',
|
||||
"key": "bookmark",
|
||||
}
|
||||
)
|
||||
|
||||
for result in json_resp['resource_response']['data']['results']:
|
||||
for result in json_resp["resource_response"]["data"]["results"]:
|
||||
|
||||
if result['type'] == 'story':
|
||||
if result["type"] == "story":
|
||||
continue
|
||||
|
||||
main_image = result['images']['orig']
|
||||
results.append(
|
||||
{
|
||||
'template': 'images.html',
|
||||
'url': result.get('link') or f"{base_url}/pin/{result['id']}/",
|
||||
'title': result.get('title') or result.get('grid_title'),
|
||||
'content': (result.get('rich_summary') or {}).get('display_description') or "",
|
||||
'img_src': main_image['url'],
|
||||
'thumbnail_src': result['images']['236x']['url'],
|
||||
'source': (result.get('rich_summary') or {}).get('site_name'),
|
||||
'resolution': f"{main_image['width']}x{main_image['height']}",
|
||||
'author': f"{result['pinner'].get('full_name')} ({result['pinner']['username']})",
|
||||
}
|
||||
main_image = result["images"]["orig"]
|
||||
|
||||
title = result.get("title") or result.get("grid_title") or ""
|
||||
if len(title) < 5:
|
||||
visual_annotation = result.get("pin_join", {}).get("visual_annotation")
|
||||
if visual_annotation:
|
||||
title = visual_annotation[0]
|
||||
else:
|
||||
title = result.get("name") or result.get("auto_alt_text") or ""
|
||||
|
||||
res.add(
|
||||
res.types.Image(
|
||||
url=result.get("link") or f"{base_url}/pin/{result['id']}/",
|
||||
title=title,
|
||||
content=(result.get("rich_summary") or {}).get("display_description") or "",
|
||||
img_src=main_image["url"],
|
||||
thumbnail_src=result["images"]["236x"]["url"],
|
||||
source=(result.get("rich_summary") or {}).get("site_name") or "",
|
||||
resolution=f"{main_image['width']}x{main_image['height']}",
|
||||
author=f"{result['pinner'].get('full_name')} ({result['pinner']['username']})",
|
||||
)
|
||||
)
|
||||
|
||||
return results
|
||||
return res
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Startpage's language & region selectors are a mess ..
|
||||
"""Startpage requires solving an Anubis POW captcha (difficulty 4).
|
||||
Solving it requires a lot of CPU, so the engine is set inactive by default.
|
||||
|
||||
Startpage's language & region selectors are a mess ..
|
||||
|
||||
.. _startpage regions:
|
||||
|
||||
@@ -84,6 +87,7 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
|
||||
"""
|
||||
# pylint: disable=too-many-statements
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import typing as t
|
||||
from collections import OrderedDict
|
||||
@@ -99,7 +103,7 @@ from searx.enginelib import EngineCache
|
||||
from searx.enginelib.traits import EngineTraits
|
||||
from searx.exceptions import SearxEngineCaptchaException
|
||||
from searx.locales import region_tag
|
||||
from searx.network import get # see https://github.com/searxng/searxng/issues/762
|
||||
from searx.network import get, post # see https://github.com/searxng/searxng/issues/762
|
||||
from searx.utils import (
|
||||
eval_xpath,
|
||||
extr,
|
||||
@@ -176,6 +180,45 @@ def setup(_: dict[str, t.Any]) -> bool | None:
|
||||
sc_code_cache_sec = 3600
|
||||
"""Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
|
||||
|
||||
# startpage's anubis difficulty is set to 4
|
||||
max_difficulty = 4
|
||||
|
||||
|
||||
def _solve_anubis(resp) -> str:
|
||||
"""Anubis POW solver"""
|
||||
payload = loads(extr(resp.text, '<script id="anubis_challenge" type="application/json">', "</script>"))
|
||||
challenge = payload["challenge"]
|
||||
difficulty = int(payload["rules"]["difficulty"])
|
||||
if difficulty > max_difficulty:
|
||||
raise SearxEngineCaptchaException(message="startpage: Anubis difficulty too high")
|
||||
prefix = "0" * difficulty
|
||||
blob = challenge["randomData"].encode()
|
||||
for nonce in range(16**difficulty * 8): # max search is 8x average search, e^-8 = 0.034% will fail
|
||||
digest = hashlib.sha256(blob + str(nonce).encode()).hexdigest()
|
||||
if digest.startswith(prefix):
|
||||
break
|
||||
else:
|
||||
raise SearxEngineCaptchaException(message="startpage: Anubis failed")
|
||||
|
||||
pass_resp = get(
|
||||
f"{base_url}/.within.website/x/cmd/anubis/api/pass-challenge",
|
||||
params={
|
||||
"id": challenge["id"],
|
||||
"response": digest,
|
||||
"nonce": nonce,
|
||||
"redir": str(resp.url),
|
||||
"elapsedTime": "1",
|
||||
},
|
||||
cookies=resp.cookies,
|
||||
allow_redirects=False,
|
||||
)
|
||||
auth = pass_resp.cookies.get("spchal-auth")
|
||||
if not auth:
|
||||
raise SearxEngineCaptchaException(message="startpage: Anubis pass-challenge failed")
|
||||
auth = str(auth)
|
||||
CACHE.set("SPCHAL_AUTH", auth, expire=240)
|
||||
return auth
|
||||
|
||||
|
||||
def get_sc_code(params):
|
||||
"""Get an actual ``sc`` argument from Startpage's search form (HTML page).
|
||||
@@ -201,6 +244,9 @@ def get_sc_code(params):
|
||||
logger.debug("get_sc_code: request headers: %s", headers)
|
||||
resp = get(get_sc_url, headers=headers)
|
||||
|
||||
if 'id="anubis_challenge"' in resp.text:
|
||||
resp = get(get_sc_url, headers=headers, cookies={"spchal-auth": _solve_anubis(resp)})
|
||||
|
||||
# ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)
|
||||
# ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg
|
||||
# ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21
|
||||
@@ -239,8 +285,8 @@ def request(query, params):
|
||||
Additionally the arguments form Startpage's search form needs to be set in
|
||||
HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.
|
||||
"""
|
||||
engine_region = traits.get_region(params["searxng_locale"], "en-US")
|
||||
engine_language = traits.get_language(params["searxng_locale"], "en")
|
||||
engine_region = traits.get_region(params["searxng_locale"], "en_US")
|
||||
engine_language = traits.get_language(params["searxng_locale"], "english")
|
||||
|
||||
params["headers"]["Origin"] = base_url
|
||||
params["headers"]["Referer"] = base_url + "/"
|
||||
@@ -262,9 +308,9 @@ def request(query, params):
|
||||
args["language"] = engine_language
|
||||
args["lui"] = engine_language
|
||||
|
||||
args["segment"] = "startpage.udog"
|
||||
if params["pageno"] > 1:
|
||||
args["page"] = params["pageno"]
|
||||
args["segment"] = "startpage.udog"
|
||||
|
||||
# Build cookie
|
||||
lang_homepage = "en"
|
||||
@@ -289,6 +335,8 @@ def request(query, params):
|
||||
cookie["search_results_region"] = engine_region
|
||||
|
||||
params["cookies"]["preferences"] = "N1N".join(["%sEEE%s" % x for x in cookie.items()])
|
||||
if auth := CACHE.get("SPCHAL_AUTH"):
|
||||
params["cookies"]["spchal-auth"] = auth
|
||||
logger.debug("cookie preferences: %s", params["cookies"]["preferences"])
|
||||
|
||||
logger.debug("data: %s", args)
|
||||
@@ -400,6 +448,18 @@ def _get_image_result(result) -> dict[str, t.Any] | None:
|
||||
|
||||
|
||||
def response(resp):
|
||||
if 'id="anubis_challenge"' in resp.text:
|
||||
params = resp.search_params
|
||||
params["cookies"]["spchal-auth"] = _solve_anubis(resp)
|
||||
resp = post(
|
||||
params["url"] or search_url,
|
||||
data=params["data"],
|
||||
headers=params["headers"],
|
||||
cookies=params["cookies"],
|
||||
)
|
||||
if 'id="anubis_challenge"' in resp.text:
|
||||
raise SearxEngineCaptchaException()
|
||||
|
||||
categ = startpage_categ.capitalize()
|
||||
results_raw = "{" + extr(resp.text, f"React.createElement(UIStartpage.AppSerp{categ}, {{", "}})") + "}}"
|
||||
|
||||
|
||||
@@ -88,6 +88,9 @@ class HTTPParams(t.TypedDict):
|
||||
impersonate: t.NotRequired[str]
|
||||
"""curl_cffi impersonate target. Default: ``chrome``."""
|
||||
|
||||
default_headers: t.NotRequired[bool]
|
||||
"""Ask curl_cffi to skip setting the impersonated browser's default headers."""
|
||||
|
||||
curl_options: t.NotRequired[dict[int, t.Any]]
|
||||
"""Any extra libcurl options for the request."""
|
||||
|
||||
@@ -169,8 +172,8 @@ class OnlineProcessor(EngineProcessor):
|
||||
"cookies": params["cookies"],
|
||||
"auth": params["auth"],
|
||||
}
|
||||
for key in ("curl_options", "impersonate"):
|
||||
if params.get(key):
|
||||
for key in ("curl_options", "impersonate", "default_headers"):
|
||||
if params.get(key) is not None:
|
||||
request_args[key] = params[key]
|
||||
|
||||
verify = params.get("verify")
|
||||
|
||||
@@ -792,6 +792,10 @@ engines:
|
||||
require_api_key: false
|
||||
results: JSON
|
||||
|
||||
- name: europepmc
|
||||
engine: europepmc
|
||||
shortcut: epmc
|
||||
|
||||
- name: erowid
|
||||
engine: xpath
|
||||
paging: true
|
||||
@@ -2339,18 +2343,21 @@ engines:
|
||||
shortcut: sp
|
||||
startpage_categ: web
|
||||
categories: [general, web]
|
||||
inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
|
||||
|
||||
- name: startpage news
|
||||
engine: startpage
|
||||
startpage_categ: news
|
||||
categories: [news, web]
|
||||
shortcut: spn
|
||||
inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
|
||||
|
||||
- name: startpage images
|
||||
engine: startpage
|
||||
startpage_categ: images
|
||||
categories: [images, web]
|
||||
shortcut: spi
|
||||
inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
|
||||
|
||||
- name: steam
|
||||
engine: steam
|
||||
|
||||
Reference in New Issue
Block a user