mirror of
https://github.com/searxng/searxng.git
synced 2026-09-11 16:56:05 +00:00
Compare commits
6 Commits
3fdc6d753a
...
dependabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4b2f750b58 | ||
|
|
ffe96f8a6f | ||
|
|
931fd9787b | ||
|
|
42e1d61296 | ||
|
|
765a9999df | ||
|
|
ba055b3e09 |
8
docs/dev/engines/online/europepmc.rst
Normal file
8
docs/dev/engines/online/europepmc.rst
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
.. _europepmc engine:
|
||||||
|
|
||||||
|
==========
|
||||||
|
Europe PMC
|
||||||
|
==========
|
||||||
|
|
||||||
|
.. automodule:: searx.engines.europepmc
|
||||||
|
:members:
|
||||||
@@ -18,11 +18,11 @@ myst-parser==5.0.0
|
|||||||
linuxdoc==20260504
|
linuxdoc==20260504
|
||||||
aiounittest==1.5.0
|
aiounittest==1.5.0
|
||||||
yamllint==1.38.0
|
yamllint==1.38.0
|
||||||
wlc==2.1.1
|
wlc==2.2.0
|
||||||
coloredlogs==15.0.1
|
coloredlogs==15.0.1
|
||||||
docutils>=0.21.2;python_version <= "3.11"
|
docutils>=0.21.2;python_version <= "3.11"
|
||||||
docutils>=0.22.4; python_version > "3.11"
|
docutils>=0.22.4; python_version > "3.11"
|
||||||
parameterized==0.9.0
|
parameterized==0.9.0
|
||||||
granian[reload]==2.8.2
|
granian[reload]==2.8.2
|
||||||
basedpyright==1.39.10
|
basedpyright==1.40.0
|
||||||
types-lxml==2026.2.16
|
types-lxml==2026.2.16
|
||||||
|
|||||||
@@ -3,11 +3,11 @@ babel==2.18.0
|
|||||||
flask-babel==4.0.0
|
flask-babel==4.0.0
|
||||||
flask==3.1.3
|
flask==3.1.3
|
||||||
jinja2==3.1.6
|
jinja2==3.1.6
|
||||||
lxml==6.1.2
|
lxml==6.1.3
|
||||||
pygments==2.21.0
|
pygments==2.21.0
|
||||||
python-dateutil==2.9.0.post0
|
python-dateutil==2.9.0.post0
|
||||||
pyyaml==6.0.3
|
pyyaml==6.0.3
|
||||||
curl_cffi==0.16.1
|
curl_cffi==0.16.3
|
||||||
valkey==6.1.1
|
valkey==6.1.1
|
||||||
markdown-it-py==4.2.0
|
markdown-it-py==4.2.0
|
||||||
msgspec==0.21.1
|
msgspec==0.21.1
|
||||||
|
|||||||
@@ -72,43 +72,21 @@ def get_locale_params(engine_region: str | None) -> dict[str, str] | None:
|
|||||||
return {"mkt": engine_region}
|
return {"mkt": engine_region}
|
||||||
|
|
||||||
|
|
||||||
def override_accept_language(params: "OnlineParams", engine_region: str | None) -> None:
|
|
||||||
"""Override the ``Accept-Language`` header.
|
|
||||||
|
|
||||||
The default header built by :py:class:`~searx.search.processors.online.OnlineProcessor`
|
|
||||||
appends ``en;q=0.3`` as a fallback language::
|
|
||||||
|
|
||||||
Accept-Language: de,de-DE;q=0.7,en;q=0.3
|
|
||||||
|
|
||||||
Bing seems to better select the results locale based on the
|
|
||||||
``Accept-Language`` value header.
|
|
||||||
|
|
||||||
This function is shared with :py:mod:`searx.engines.bing_images`,
|
|
||||||
:py:mod:`searx.engines.bing_news`, and :py:mod:`searx.engines.bing_videos`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
if not engine_region or engine_region == "clear":
|
|
||||||
return
|
|
||||||
|
|
||||||
lang = engine_region.split("-")[0]
|
|
||||||
params["headers"]["Accept-Language"] = f"{engine_region},{lang};q=0.9"
|
|
||||||
|
|
||||||
|
|
||||||
def request(query: str, params: "OnlineParams"):
|
def request(query: str, params: "OnlineParams"):
|
||||||
"""Assemble a Bing-Web request."""
|
"""Assemble a Bing-Web request."""
|
||||||
|
|
||||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||||
|
|
||||||
override_accept_language(params, engine_region)
|
|
||||||
|
|
||||||
query_params: dict[str, str | int] = {
|
query_params: dict[str, str | int] = {
|
||||||
"q": query,
|
"q": query,
|
||||||
"adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
|
"adlt": _safesearch_map.get(params.get("safesearch", 0), "off"),
|
||||||
}
|
}
|
||||||
|
|
||||||
locale_params = get_locale_params(engine_region)
|
if engine_region and engine_region != "clear":
|
||||||
if locale_params:
|
lang, _, cc = engine_region.partition("-")
|
||||||
query_params.update(locale_params)
|
query_params["setlang"] = lang
|
||||||
|
if cc and cc not in ("us", "cn", "ru"): # bing just sends junk for these
|
||||||
|
query_params["cc"] = cc
|
||||||
|
|
||||||
params["url"] = f"{base_url}/search?{urlencode(query_params)}"
|
params["url"] = f"{base_url}/search?{urlencode(query_params)}"
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ from lxml import html
|
|||||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||||
fetch_traits,
|
fetch_traits,
|
||||||
get_locale_params,
|
get_locale_params,
|
||||||
override_accept_language,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# about
|
# about
|
||||||
@@ -44,8 +43,6 @@ def request(query, params):
|
|||||||
|
|
||||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||||
|
|
||||||
override_accept_language(params, engine_region)
|
|
||||||
|
|
||||||
# build URL query
|
# build URL query
|
||||||
# - example: https://www.bing.com/images/async?q=foo&async=1&first=1&count=35
|
# - example: https://www.bing.com/images/async?q=foo&async=1&first=1&count=35
|
||||||
query_params = {
|
query_params = {
|
||||||
|
|||||||
@@ -12,10 +12,7 @@ from urllib.parse import urlencode
|
|||||||
from lxml import html
|
from lxml import html
|
||||||
|
|
||||||
from searx.enginelib.traits import EngineTraits
|
from searx.enginelib.traits import EngineTraits
|
||||||
from searx.engines.bing import (
|
from searx.engines.bing import get_locale_params
|
||||||
get_locale_params,
|
|
||||||
override_accept_language,
|
|
||||||
)
|
|
||||||
from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
|
from searx.utils import eval_xpath, eval_xpath_getindex, eval_xpath_list, extract_text
|
||||||
|
|
||||||
# about
|
# about
|
||||||
@@ -54,8 +51,6 @@ def request(query, params):
|
|||||||
|
|
||||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||||
|
|
||||||
override_accept_language(params, engine_region)
|
|
||||||
|
|
||||||
# build URL query
|
# build URL query
|
||||||
# - example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
|
# - example: https://www.bing.com/news/infinitescrollajax?q=london&first=1
|
||||||
page = int(params.get("pageno", 1)) - 1
|
page = int(params.get("pageno", 1)) - 1
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ from lxml import html
|
|||||||
from searx.engines.bing import ( # pylint: disable=unused-import
|
from searx.engines.bing import ( # pylint: disable=unused-import
|
||||||
fetch_traits,
|
fetch_traits,
|
||||||
get_locale_params,
|
get_locale_params,
|
||||||
override_accept_language,
|
|
||||||
)
|
)
|
||||||
from searx.engines.bing_images import time_map
|
from searx.engines.bing_images import time_map
|
||||||
from searx.utils import eval_xpath, eval_xpath_getindex
|
from searx.utils import eval_xpath, eval_xpath_getindex
|
||||||
@@ -39,8 +38,6 @@ def request(query, params):
|
|||||||
|
|
||||||
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
engine_region = traits.get_region(params["searxng_locale"], traits.all_locale)
|
||||||
|
|
||||||
override_accept_language(params, engine_region)
|
|
||||||
|
|
||||||
# build URL query
|
# build URL query
|
||||||
# - example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
|
# - example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35
|
||||||
query_params = {
|
query_params = {
|
||||||
|
|||||||
@@ -141,12 +141,13 @@ def response(resp: "SXNG_Response") -> EngineResults:
|
|||||||
if name:
|
if name:
|
||||||
authors.add(name)
|
authors.add(name)
|
||||||
|
|
||||||
|
tag = result.get("fieldOfStudy")
|
||||||
res.add(
|
res.add(
|
||||||
res.types.Paper(
|
res.types.Paper(
|
||||||
title=result.get("title"),
|
title=result.get("title"),
|
||||||
url=url,
|
url=url,
|
||||||
content=result.get("fullText", "") or "",
|
content=result.get("fullText", "") or "",
|
||||||
tags=result.get("fieldOfStudy", []),
|
tags=[tag] if tag else [],
|
||||||
publishedDate=published_date,
|
publishedDate=published_date,
|
||||||
type=result.get("documentType", "") or "",
|
type=result.get("documentType", "") or "",
|
||||||
authors=authors,
|
authors=authors,
|
||||||
|
|||||||
@@ -14,11 +14,12 @@ can't build it ourselves and must scrape it from the HTML pages.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import typing as t
|
import typing as t
|
||||||
|
import re
|
||||||
|
|
||||||
from urllib.parse import quote_plus
|
from urllib.parse import quote_plus, urljoin
|
||||||
from lxml import html
|
from lxml import html
|
||||||
|
|
||||||
from searx.utils import html_to_text, gen_useragent, extract_text, eval_xpath
|
from searx.utils import html_to_text, extract_text, eval_xpath
|
||||||
from searx.result_types import EngineResults
|
from searx.result_types import EngineResults
|
||||||
from searx.enginelib import EngineCache
|
from searx.enginelib import EngineCache
|
||||||
from searx.network import get
|
from searx.network import get
|
||||||
@@ -38,7 +39,6 @@ about = {
|
|||||||
# engine dependent config
|
# engine dependent config
|
||||||
categories = ["general"]
|
categories = ["general"]
|
||||||
paging = True
|
paging = True
|
||||||
_HTTP_User_Agent: str = gen_useragent()
|
|
||||||
|
|
||||||
base_url = "https://duckduckgo.com"
|
base_url = "https://duckduckgo.com"
|
||||||
|
|
||||||
@@ -73,6 +73,8 @@ def _fetch_first_page_link(
|
|||||||
resp = get(
|
resp = get(
|
||||||
url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
|
url=f"{base_url}/?q={quote_plus(query)}&t=h_&ia=web",
|
||||||
headers=headers,
|
headers=headers,
|
||||||
|
impersonate="firefox",
|
||||||
|
default_headers=False,
|
||||||
timeout=2,
|
timeout=2,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -96,6 +98,43 @@ def _cache_key(query: str, pageno: int) -> str:
|
|||||||
return f"nextpage_url|{query}|{pageno}"
|
return f"nextpage_url|{query}|{pageno}"
|
||||||
|
|
||||||
|
|
||||||
|
def _solve_jsa(resp: "SXNG_Response") -> "SXNG_Response":
|
||||||
|
"""Duckduckgo sometimes issues a challenge instead of json."""
|
||||||
|
|
||||||
|
# length that a real browser would report for where the broken snippet is
|
||||||
|
html_len = {
|
||||||
|
"<p><div></p><p></div": 32,
|
||||||
|
"<li><div></li><li></div": 29,
|
||||||
|
"<div><div></div><div></div": 33,
|
||||||
|
"<br><div></br><br></div": 23,
|
||||||
|
}
|
||||||
|
|
||||||
|
js = resp.text or ""
|
||||||
|
jsa_match = re.search(r"let jsa = (\d+);.*?DDG\.deep\.initialize\('([^']+)'", js, re.S)
|
||||||
|
if not jsa_match:
|
||||||
|
return resp
|
||||||
|
|
||||||
|
js_functions = dict(re.findall(r"let (\w+) = function\(num\) \{([^}]*)\};", js))
|
||||||
|
jsa = int(jsa_match.group(1))
|
||||||
|
try:
|
||||||
|
for name in re.findall(r"jsa = (\w+)\(jsa\);", js):
|
||||||
|
body = js_functions[name]
|
||||||
|
mul = re.search(r"num \* (\d+)", body)
|
||||||
|
jsa = jsa * int(mul.group(1)) if mul else jsa + html_len[re.search(r"`([^`]+)`", body).group(1)]
|
||||||
|
except (KeyError, AttributeError):
|
||||||
|
return resp
|
||||||
|
|
||||||
|
params = resp.search_params
|
||||||
|
follow = get(
|
||||||
|
urljoin("https://links.duckduckgo.com", jsa_match.group(2) + str(jsa)),
|
||||||
|
headers=params["headers"],
|
||||||
|
impersonate="firefox",
|
||||||
|
default_headers=False,
|
||||||
|
)
|
||||||
|
follow.search_params = params
|
||||||
|
return follow
|
||||||
|
|
||||||
|
|
||||||
def request(query: str, params: "OnlineParams") -> None:
|
def request(query: str, params: "OnlineParams") -> None:
|
||||||
|
|
||||||
if len(query) >= 500:
|
if len(query) >= 500:
|
||||||
@@ -103,25 +142,15 @@ def request(query: str, params: "OnlineParams") -> None:
|
|||||||
params["url"] = None
|
params["url"] = None
|
||||||
return
|
return
|
||||||
|
|
||||||
headers = params["headers"]
|
# firefox TLS only
|
||||||
|
params["impersonate"] = "firefox"
|
||||||
# The vqd value is generated from the query and the UA header. To be able
|
params["default_headers"] = False
|
||||||
# to reuse the vqd value, the UA header must be static.
|
|
||||||
headers["User-Agent"] = _HTTP_User_Agent
|
|
||||||
headers["Accept"] = "*/*"
|
|
||||||
headers["Referer"] = f"{base_url}/"
|
|
||||||
headers["Host"] = "duckduckgo.com"
|
|
||||||
|
|
||||||
# Sec-Fetch headers are required to not get blocked when sending a Firefox user agent
|
|
||||||
headers["Sec-Fetch-Dest"] = "script"
|
|
||||||
headers["Sec-Fetch-Mode"] = "no-cors"
|
|
||||||
headers["Sec-Fetch-Site"] = "same-site"
|
|
||||||
|
|
||||||
api_url = ""
|
api_url = ""
|
||||||
if params["pageno"] > 1:
|
if params["pageno"] > 1:
|
||||||
api_url = CACHE.get(_cache_key(query, params["pageno"]))
|
api_url = CACHE.get(_cache_key(query, params["pageno"]))
|
||||||
else:
|
else:
|
||||||
api_url = _fetch_first_page_link(query, headers)
|
api_url = _fetch_first_page_link(query, params["headers"])
|
||||||
|
|
||||||
if not api_url:
|
if not api_url:
|
||||||
params["url"] = None
|
params["url"] = None
|
||||||
@@ -129,14 +158,27 @@ def request(query: str, params: "OnlineParams") -> None:
|
|||||||
|
|
||||||
params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
|
params["url"] = api_url.replace("/d.js?", "/d.js?o=json&")
|
||||||
|
|
||||||
|
# loads as a script
|
||||||
|
headers = params["headers"]
|
||||||
|
headers["Accept"] = "*/*"
|
||||||
|
headers["Sec-Fetch-Dest"] = "script"
|
||||||
|
headers["Sec-Fetch-Mode"] = "no-cors"
|
||||||
|
headers["Sec-Fetch-Site"] = "same-site"
|
||||||
|
headers["Referer"] = f"{base_url}/"
|
||||||
|
|
||||||
# TODO: support safesearch, timerange and engine traits # pylint:disable=fixme
|
# TODO: support safesearch, timerange and engine traits # pylint:disable=fixme
|
||||||
|
|
||||||
|
|
||||||
def response(resp: "SXNG_Response"):
|
def response(resp: "SXNG_Response"):
|
||||||
res = EngineResults()
|
res = EngineResults()
|
||||||
res_json = resp.json()
|
|
||||||
|
|
||||||
for result in res_json["results"]:
|
# check if ddg returns a challenge
|
||||||
|
# e.g. 'site:github.com searxng'
|
||||||
|
if "let jsa =" in (resp.text or ""):
|
||||||
|
resp = _solve_jsa(resp)
|
||||||
|
|
||||||
|
results = resp.json()["results"]
|
||||||
|
for result in results:
|
||||||
if "u" not in result:
|
if "u" not in result:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -144,13 +186,13 @@ def response(resp: "SXNG_Response"):
|
|||||||
res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
|
res.types.MainResult(url=result["u"], title=html_to_text(result["t"]), content=html_to_text(result["a"]))
|
||||||
)
|
)
|
||||||
|
|
||||||
# link to next page
|
if results:
|
||||||
next_page_path = res_json["results"][-1].get("n")
|
next_page_path = results[-1].get("n")
|
||||||
if next_page_path:
|
if next_page_path:
|
||||||
CACHE.set(
|
CACHE.set(
|
||||||
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
|
_cache_key(resp.search_params["query"], resp.search_params["pageno"] + 1),
|
||||||
base_url + next_page_path,
|
base_url + next_page_path,
|
||||||
expire=60 * 60,
|
expire=60 * 60,
|
||||||
)
|
)
|
||||||
|
|
||||||
return res
|
return res
|
||||||
|
|||||||
150
searx/engines/europepmc.py
Normal file
150
searx/engines/europepmc.py
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
"""`Europe PMC`_ provides comprehensive access to life sciences literature from
|
||||||
|
trusted sources. With Europe PMC you can search and read millions of
|
||||||
|
publications, preprints and other documents enriched with links to supporting
|
||||||
|
data, reviews, protocols, and other relevant resources.
|
||||||
|
|
||||||
|
.. _Europe PMC: https://europepmc.org/
|
||||||
|
|
||||||
|
Configuration
|
||||||
|
=============
|
||||||
|
|
||||||
|
.. code:: yaml
|
||||||
|
|
||||||
|
- name: europepmc
|
||||||
|
engine: europepmc
|
||||||
|
shortcut: epmc
|
||||||
|
|
||||||
|
Implementations
|
||||||
|
===============
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import typing as t
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
|
from dateutil.parser import isoparse
|
||||||
|
|
||||||
|
from searx.enginelib import EngineCache
|
||||||
|
from searx.result_types import EngineResults
|
||||||
|
from searx.utils import html_to_text
|
||||||
|
|
||||||
|
if t.TYPE_CHECKING:
|
||||||
|
from searx.extended_types import SXNG_Response
|
||||||
|
from searx.search.processors import OnlineParams
|
||||||
|
|
||||||
|
|
||||||
|
about = {
|
||||||
|
"website": "https://europepmc.org/",
|
||||||
|
"wikidata_id": "Q5412157",
|
||||||
|
"official_api_documentation": "https://europepmc.org/RestfulWebService",
|
||||||
|
"use_official_api": True,
|
||||||
|
"require_api_key": False,
|
||||||
|
"results": "JSON",
|
||||||
|
}
|
||||||
|
|
||||||
|
categories = ["science", "scientific publications"]
|
||||||
|
paging = True
|
||||||
|
|
||||||
|
# engine dependent config
|
||||||
|
search_url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search"
|
||||||
|
article_url = "https://europepmc.org/article/"
|
||||||
|
|
||||||
|
page_size = 20
|
||||||
|
|
||||||
|
CACHE: EngineCache
|
||||||
|
"""Cache for storing the pagination cursor."""
|
||||||
|
|
||||||
|
|
||||||
|
def setup(engine_settings: dict[str, t.Any]):
|
||||||
|
global CACHE # pylint: disable=global-statement
|
||||||
|
CACHE = EngineCache(engine_settings["name"])
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_key(query: str, page: int) -> str:
|
||||||
|
return f"{query}|{page}"
|
||||||
|
|
||||||
|
|
||||||
|
def request(query: str, params: "OnlineParams") -> None:
|
||||||
|
args = {
|
||||||
|
"query": query,
|
||||||
|
"format": "json",
|
||||||
|
"resultType": "core",
|
||||||
|
"pageSize": page_size,
|
||||||
|
}
|
||||||
|
|
||||||
|
if params["pageno"] > 1:
|
||||||
|
if cursor := CACHE.get(_cache_key(query, params["pageno"])):
|
||||||
|
args["cursorMark"] = cursor
|
||||||
|
else:
|
||||||
|
# no cached cursor for that page
|
||||||
|
params["url"] = None
|
||||||
|
return
|
||||||
|
|
||||||
|
params["url"] = f"{search_url}?{urlencode(args)}"
|
||||||
|
|
||||||
|
|
||||||
|
def response(resp: "SXNG_Response") -> EngineResults:
|
||||||
|
res = EngineResults()
|
||||||
|
|
||||||
|
json_resp = resp.json()
|
||||||
|
|
||||||
|
# store pagination cursor for loading next pages in cache
|
||||||
|
if next_cursor := json_resp.get("nextCursorMark"):
|
||||||
|
next_page = resp.search_params["pageno"] + 1
|
||||||
|
query = resp.search_params["query"]
|
||||||
|
CACHE.set(_cache_key(query, next_page), next_cursor)
|
||||||
|
|
||||||
|
all_results = json_resp.get("resultList", {}).get("result", [])
|
||||||
|
|
||||||
|
for item in all_results:
|
||||||
|
source = item.get("source", "")
|
||||||
|
identifier = item.get("id", "")
|
||||||
|
url = f"{article_url}{source}/{identifier}" if source and identifier else ""
|
||||||
|
|
||||||
|
journal_info: dict[str, t.Any] = item.get("journalInfo", {})
|
||||||
|
journal: dict[str, t.Any] = journal_info.get("journal", {})
|
||||||
|
|
||||||
|
res.add(
|
||||||
|
res.types.Paper(
|
||||||
|
url=url,
|
||||||
|
title=html_to_text(item.get("title", "")),
|
||||||
|
content=html_to_text(item.get("abstractText", "")),
|
||||||
|
journal=journal.get("title", ""),
|
||||||
|
issn=[journal.get("issn", "")],
|
||||||
|
authors=_get_authors(item),
|
||||||
|
doi=item.get("doi", ""),
|
||||||
|
publishedDate=_get_published_date(item),
|
||||||
|
type=", ".join((item.get("pubTypeList", {})).get("pubType", [])),
|
||||||
|
pdf_url=_get_pdf_url(item),
|
||||||
|
html_url=url,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
def _get_authors(item: dict[str, t.Any]) -> list:
|
||||||
|
"""Extract the list of authors from the item."""
|
||||||
|
if authors := item.get("authorString", None):
|
||||||
|
authors = [author.strip().rstrip(".") for author in authors.split(",") if author.strip()]
|
||||||
|
else:
|
||||||
|
authors = []
|
||||||
|
return authors
|
||||||
|
|
||||||
|
|
||||||
|
def _get_pdf_url(item: dict[str, t.Any]) -> str:
|
||||||
|
"""Extract the PDF URL in case it is open access."""
|
||||||
|
for url_info in (item.get("fullTextUrlList", {})).get("fullTextUrl", []):
|
||||||
|
if url_info.get("documentStyle") == "pdf" and url_info.get("availabilityCode") == "OA":
|
||||||
|
return url_info.get("url", "")
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _get_published_date(item: dict[str, t.Any]) -> datetime | None:
|
||||||
|
"""Extract the published date from the item and convert it to a datetime object."""
|
||||||
|
if unformatted_date := item.get("firstPublicationDate"):
|
||||||
|
return isoparse(unformatted_date)
|
||||||
|
return None
|
||||||
@@ -1,5 +1,8 @@
|
|||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
"""Startpage's language & region selectors are a mess ..
|
"""Startpage requires solving an Anubis POW captcha (difficulty 4).
|
||||||
|
Solving it requires a lot of CPU, so the engine is set inactive by default.
|
||||||
|
|
||||||
|
Startpage's language & region selectors are a mess ..
|
||||||
|
|
||||||
.. _startpage regions:
|
.. _startpage regions:
|
||||||
|
|
||||||
@@ -84,6 +87,7 @@ Startpage's category (for Web-search, News, Videos, ..) is set by
|
|||||||
"""
|
"""
|
||||||
# pylint: disable=too-many-statements
|
# pylint: disable=too-many-statements
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import re
|
import re
|
||||||
import typing as t
|
import typing as t
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
@@ -99,7 +103,7 @@ from searx.enginelib import EngineCache
|
|||||||
from searx.enginelib.traits import EngineTraits
|
from searx.enginelib.traits import EngineTraits
|
||||||
from searx.exceptions import SearxEngineCaptchaException
|
from searx.exceptions import SearxEngineCaptchaException
|
||||||
from searx.locales import region_tag
|
from searx.locales import region_tag
|
||||||
from searx.network import get # see https://github.com/searxng/searxng/issues/762
|
from searx.network import get, post # see https://github.com/searxng/searxng/issues/762
|
||||||
from searx.utils import (
|
from searx.utils import (
|
||||||
eval_xpath,
|
eval_xpath,
|
||||||
extr,
|
extr,
|
||||||
@@ -176,6 +180,45 @@ def setup(_: dict[str, t.Any]) -> bool | None:
|
|||||||
sc_code_cache_sec = 3600
|
sc_code_cache_sec = 3600
|
||||||
"""Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
|
"""Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""
|
||||||
|
|
||||||
|
# startpage's anubis difficulty is set to 4
|
||||||
|
max_difficulty = 4
|
||||||
|
|
||||||
|
|
||||||
|
def _solve_anubis(resp) -> str:
|
||||||
|
"""Anubis POW solver"""
|
||||||
|
payload = loads(extr(resp.text, '<script id="anubis_challenge" type="application/json">', "</script>"))
|
||||||
|
challenge = payload["challenge"]
|
||||||
|
difficulty = int(payload["rules"]["difficulty"])
|
||||||
|
if difficulty > max_difficulty:
|
||||||
|
raise SearxEngineCaptchaException(message="startpage: Anubis difficulty too high")
|
||||||
|
prefix = "0" * difficulty
|
||||||
|
blob = challenge["randomData"].encode()
|
||||||
|
for nonce in range(16**difficulty * 8): # max search is 8x average search, e^-8 = 0.034% will fail
|
||||||
|
digest = hashlib.sha256(blob + str(nonce).encode()).hexdigest()
|
||||||
|
if digest.startswith(prefix):
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
raise SearxEngineCaptchaException(message="startpage: Anubis failed")
|
||||||
|
|
||||||
|
pass_resp = get(
|
||||||
|
f"{base_url}/.within.website/x/cmd/anubis/api/pass-challenge",
|
||||||
|
params={
|
||||||
|
"id": challenge["id"],
|
||||||
|
"response": digest,
|
||||||
|
"nonce": nonce,
|
||||||
|
"redir": str(resp.url),
|
||||||
|
"elapsedTime": "1",
|
||||||
|
},
|
||||||
|
cookies=resp.cookies,
|
||||||
|
allow_redirects=False,
|
||||||
|
)
|
||||||
|
auth = pass_resp.cookies.get("spchal-auth")
|
||||||
|
if not auth:
|
||||||
|
raise SearxEngineCaptchaException(message="startpage: Anubis pass-challenge failed")
|
||||||
|
auth = str(auth)
|
||||||
|
CACHE.set("SPCHAL_AUTH", auth, expire=240)
|
||||||
|
return auth
|
||||||
|
|
||||||
|
|
||||||
def get_sc_code(params):
|
def get_sc_code(params):
|
||||||
"""Get an actual ``sc`` argument from Startpage's search form (HTML page).
|
"""Get an actual ``sc`` argument from Startpage's search form (HTML page).
|
||||||
@@ -201,6 +244,9 @@ def get_sc_code(params):
|
|||||||
logger.debug("get_sc_code: request headers: %s", headers)
|
logger.debug("get_sc_code: request headers: %s", headers)
|
||||||
resp = get(get_sc_url, headers=headers)
|
resp = get(get_sc_url, headers=headers)
|
||||||
|
|
||||||
|
if 'id="anubis_challenge"' in resp.text:
|
||||||
|
resp = get(get_sc_url, headers=headers, cookies={"spchal-auth": _solve_anubis(resp)})
|
||||||
|
|
||||||
# ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)
|
# ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)
|
||||||
# ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg
|
# ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg
|
||||||
# ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21
|
# ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21
|
||||||
@@ -239,8 +285,8 @@ def request(query, params):
|
|||||||
Additionally the arguments form Startpage's search form needs to be set in
|
Additionally the arguments form Startpage's search form needs to be set in
|
||||||
HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.
|
HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.
|
||||||
"""
|
"""
|
||||||
engine_region = traits.get_region(params["searxng_locale"], "en-US")
|
engine_region = traits.get_region(params["searxng_locale"], "en_US")
|
||||||
engine_language = traits.get_language(params["searxng_locale"], "en")
|
engine_language = traits.get_language(params["searxng_locale"], "english")
|
||||||
|
|
||||||
params["headers"]["Origin"] = base_url
|
params["headers"]["Origin"] = base_url
|
||||||
params["headers"]["Referer"] = base_url + "/"
|
params["headers"]["Referer"] = base_url + "/"
|
||||||
@@ -262,9 +308,9 @@ def request(query, params):
|
|||||||
args["language"] = engine_language
|
args["language"] = engine_language
|
||||||
args["lui"] = engine_language
|
args["lui"] = engine_language
|
||||||
|
|
||||||
|
args["segment"] = "startpage.udog"
|
||||||
if params["pageno"] > 1:
|
if params["pageno"] > 1:
|
||||||
args["page"] = params["pageno"]
|
args["page"] = params["pageno"]
|
||||||
args["segment"] = "startpage.udog"
|
|
||||||
|
|
||||||
# Build cookie
|
# Build cookie
|
||||||
lang_homepage = "en"
|
lang_homepage = "en"
|
||||||
@@ -289,6 +335,8 @@ def request(query, params):
|
|||||||
cookie["search_results_region"] = engine_region
|
cookie["search_results_region"] = engine_region
|
||||||
|
|
||||||
params["cookies"]["preferences"] = "N1N".join(["%sEEE%s" % x for x in cookie.items()])
|
params["cookies"]["preferences"] = "N1N".join(["%sEEE%s" % x for x in cookie.items()])
|
||||||
|
if auth := CACHE.get("SPCHAL_AUTH"):
|
||||||
|
params["cookies"]["spchal-auth"] = auth
|
||||||
logger.debug("cookie preferences: %s", params["cookies"]["preferences"])
|
logger.debug("cookie preferences: %s", params["cookies"]["preferences"])
|
||||||
|
|
||||||
logger.debug("data: %s", args)
|
logger.debug("data: %s", args)
|
||||||
@@ -400,6 +448,18 @@ def _get_image_result(result) -> dict[str, t.Any] | None:
|
|||||||
|
|
||||||
|
|
||||||
def response(resp):
|
def response(resp):
|
||||||
|
if 'id="anubis_challenge"' in resp.text:
|
||||||
|
params = resp.search_params
|
||||||
|
params["cookies"]["spchal-auth"] = _solve_anubis(resp)
|
||||||
|
resp = post(
|
||||||
|
params["url"] or search_url,
|
||||||
|
data=params["data"],
|
||||||
|
headers=params["headers"],
|
||||||
|
cookies=params["cookies"],
|
||||||
|
)
|
||||||
|
if 'id="anubis_challenge"' in resp.text:
|
||||||
|
raise SearxEngineCaptchaException()
|
||||||
|
|
||||||
categ = startpage_categ.capitalize()
|
categ = startpage_categ.capitalize()
|
||||||
results_raw = "{" + extr(resp.text, f"React.createElement(UIStartpage.AppSerp{categ}, {{", "}})") + "}}"
|
results_raw = "{" + extr(resp.text, f"React.createElement(UIStartpage.AppSerp{categ}, {{", "}})") + "}}"
|
||||||
|
|
||||||
|
|||||||
@@ -88,6 +88,9 @@ class HTTPParams(t.TypedDict):
|
|||||||
impersonate: t.NotRequired[str]
|
impersonate: t.NotRequired[str]
|
||||||
"""curl_cffi impersonate target. Default: ``chrome``."""
|
"""curl_cffi impersonate target. Default: ``chrome``."""
|
||||||
|
|
||||||
|
default_headers: t.NotRequired[bool]
|
||||||
|
"""Ask curl_cffi to skip setting the impersonated browser's default headers."""
|
||||||
|
|
||||||
curl_options: t.NotRequired[dict[int, t.Any]]
|
curl_options: t.NotRequired[dict[int, t.Any]]
|
||||||
"""Any extra libcurl options for the request."""
|
"""Any extra libcurl options for the request."""
|
||||||
|
|
||||||
@@ -169,8 +172,8 @@ class OnlineProcessor(EngineProcessor):
|
|||||||
"cookies": params["cookies"],
|
"cookies": params["cookies"],
|
||||||
"auth": params["auth"],
|
"auth": params["auth"],
|
||||||
}
|
}
|
||||||
for key in ("curl_options", "impersonate"):
|
for key in ("curl_options", "impersonate", "default_headers"):
|
||||||
if params.get(key):
|
if params.get(key) is not None:
|
||||||
request_args[key] = params[key]
|
request_args[key] = params[key]
|
||||||
|
|
||||||
verify = params.get("verify")
|
verify = params.get("verify")
|
||||||
|
|||||||
@@ -792,6 +792,10 @@ engines:
|
|||||||
require_api_key: false
|
require_api_key: false
|
||||||
results: JSON
|
results: JSON
|
||||||
|
|
||||||
|
- name: europepmc
|
||||||
|
engine: europepmc
|
||||||
|
shortcut: epmc
|
||||||
|
|
||||||
- name: erowid
|
- name: erowid
|
||||||
engine: xpath
|
engine: xpath
|
||||||
paging: true
|
paging: true
|
||||||
@@ -2339,18 +2343,21 @@ engines:
|
|||||||
shortcut: sp
|
shortcut: sp
|
||||||
startpage_categ: web
|
startpage_categ: web
|
||||||
categories: [general, web]
|
categories: [general, web]
|
||||||
|
inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
|
||||||
|
|
||||||
- name: startpage news
|
- name: startpage news
|
||||||
engine: startpage
|
engine: startpage
|
||||||
startpage_categ: news
|
startpage_categ: news
|
||||||
categories: [news, web]
|
categories: [news, web]
|
||||||
shortcut: spn
|
shortcut: spn
|
||||||
|
inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
|
||||||
|
|
||||||
- name: startpage images
|
- name: startpage images
|
||||||
engine: startpage
|
engine: startpage
|
||||||
startpage_categ: images
|
startpage_categ: images
|
||||||
categories: [images, web]
|
categories: [images, web]
|
||||||
shortcut: spi
|
shortcut: spi
|
||||||
|
inactive: true # uses a Proof Of Work captcha https://github.com/searxng/searxng/pull/6669
|
||||||
|
|
||||||
- name: steam
|
- name: steam
|
||||||
engine: steam
|
engine: steam
|
||||||
|
|||||||
Reference in New Issue
Block a user