[doc] build from commit 97814c62a8
4
.buildinfo
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
# Sphinx build info version 1
|
||||||
|
# This file records the configuration used when building these files. When it is not found, a full rebuild will be done.
|
||||||
|
config: 094f4f45c33c27474020219ae76e488c
|
||||||
|
tags: 645f666f9bcd5a90fca523b33c5a78b7
|
||||||
98
404.html
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="./">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>Page not found — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="/_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="/_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="/_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="/_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="/_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="/_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="/genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="/search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="/genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="/py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="/index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">Page not found</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Page not found</h1>
|
||||||
|
|
||||||
|
Unfortunately we couldn't find the content you were looking for.
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="/index.html">
|
||||||
|
<img class="logo" src="/_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="/index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="/user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="/own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="/admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="/dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="/utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="/src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="/index.html">Overview</a>
|
||||||
|
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="/search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
1438
_downloads/ad0ebe55d6b53b1559e0ca8dee6f30b9/reST.rst
Normal file
13
_images/SVG-1fb7029fa2cc454a267bae271cccb2c591387416.svg
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" version="1.1"
|
||||||
|
baseProfile="full" width="70px" height="40px"
|
||||||
|
viewBox="0 0 700 400"
|
||||||
|
>
|
||||||
|
<line x1="180" y1="370"
|
||||||
|
x2="500" y2="50"
|
||||||
|
stroke="black" stroke-width="15px"
|
||||||
|
/>
|
||||||
|
<polygon points="585 0 525 25 585 50"
|
||||||
|
transform="rotate(135 525 25)"
|
||||||
|
/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 385 B |
31
_images/arch_public.dot
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
digraph G {
|
||||||
|
|
||||||
|
node [style=filled, shape=box, fillcolor="#ffffcc", fontname=Sans];
|
||||||
|
edge [fontname="Sans"];
|
||||||
|
|
||||||
|
browser [label="browser", shape=tab, fillcolor=aliceblue];
|
||||||
|
rp [label="reverse proxy"];
|
||||||
|
static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray];
|
||||||
|
uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searxng.sh.html"]
|
||||||
|
valkey [label="valkey DB", shape=cylinder];
|
||||||
|
|
||||||
|
searxng1 [label="SearXNG #1", fontcolor=blue3];
|
||||||
|
searxng2 [label="SearXNG #2", fontcolor=blue3];
|
||||||
|
searxng3 [label="SearXNG #3", fontcolor=blue3];
|
||||||
|
searxng4 [label="SearXNG #4", fontcolor=blue3];
|
||||||
|
|
||||||
|
browser -> rp [label="HTTPS"]
|
||||||
|
|
||||||
|
subgraph cluster_searxng {
|
||||||
|
label = "SearXNG instance" fontname=Sans;
|
||||||
|
bgcolor="#fafafa";
|
||||||
|
{ rank=same; static rp };
|
||||||
|
rp -> static [label="optional: reverse proxy serves static files", fillcolor=slategray, fontcolor=slategray];
|
||||||
|
rp -> uwsgi [label="http:// (tcp) or unix:// (socket)"];
|
||||||
|
uwsgi -> searxng1 -> valkey;
|
||||||
|
uwsgi -> searxng2 -> valkey;
|
||||||
|
uwsgi -> searxng3 -> valkey;
|
||||||
|
uwsgi -> searxng4 -> valkey;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
1
_images/browserstack.svg
Normal file
|
After Width: | Height: | Size: 7.2 KiB |
1
_images/docker.svg
Normal file
|
After Width: | Height: | Size: 5.9 KiB |
BIN
_images/ffox-setting-proxy-socks.png
Normal file
|
After Width: | Height: | Size: 59 KiB |
3
_images/hello.dot
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
graph G {
|
||||||
|
Hello -- World
|
||||||
|
}
|
||||||
10
_images/svg_image.svg
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!-- originate: https://commons.wikimedia.org/wiki/File:Variable_Resistor.svg -->
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg"
|
||||||
|
version="1.1" baseProfile="full"
|
||||||
|
width="70px" height="40px" viewBox="0 0 700 400">
|
||||||
|
<line x1="0" y1="200" x2="700" y2="200" stroke="black" stroke-width="20px"/>
|
||||||
|
<rect x="100" y="100" width="500" height="200" fill="white" stroke="black" stroke-width="20px"/>
|
||||||
|
<line x1="180" y1="370" x2="500" y2="50" stroke="black" stroke-width="15px"/>
|
||||||
|
<polygon points="585 0 525 25 585 50" transform="rotate(135 525 25)"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 580 B |
47
_images/translation.svg
Normal file
|
After Width: | Height: | Size: 18 KiB |
1
_images/tuta.svg
Normal file
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 1024 384"><path fill="#410002" d="M479.178 119.294c-.533-.016-.998.357-1.218 1.078l-24.438 78.364.005-.004c-8.59 27.537 4.516 46.485 34.268 46.485 4.336 0 10.006-.357 11.776-.797.885-.264 1.33-.71 1.594-1.506l5.134-17.177c.264-.973-.09-1.77-1.507-1.683-4.517.445-8.588.797-12.308.797-14.964 0-21.075-7.968-16.646-22.224l10.446-33.47h30.373c.797 0 1.506-.445 1.858-1.418l5.401-17.355c.264-.973-.264-1.681-1.417-1.681H492.66l3.895-12.662c.265-.885.089-1.418-.62-2.034l-15.761-14.255c-.332-.3-.676-.45-.996-.459zm173.64 0c-.532-.016-.996.357-1.218 1.078l-24.436 78.364.005-.004c-8.59 27.537 4.517 46.485 34.268 46.485 4.342 0 10.004-.357 11.778-.797.884-.264 1.324-.71 1.593-1.506l5.133-17.177c.26-.973 0-1.77-1.508-1.683-4.517.445-8.59.797-12.307.797-14.966 0-21.077-7.968-16.646-22.224l10.445-33.47H690.3c.795 0 1.504-.445 1.854-1.418l5.402-17.355c.265-.973-.26-1.681-1.414-1.681H666.3l3.896-12.662c.265-.885.087-1.418-.618-2.034l-15.765-14.255c-.332-.3-.676-.45-.996-.459zm-48.32 29.404c-.974 0-1.503.444-1.862 1.417L590.502 188.9c-7.525 23.998-19.478 37.721-31.965 37.721-12.487 0-17.797-9.83-13.105-24.883l16.028-51.178c.351-1.149-.088-1.857-1.328-1.857H539.94c-.97 0-1.505.444-1.86 1.417l-15.497 49.233.008-.005c-8.765 27.982 5.315 46.31 27.452 46.31 12.747 0 22.756-6.111 29.93-16.118l-.176 12.838c0 1.241.621 1.593 1.681 1.593h14.17c1.064 0 1.504-.445 1.859-1.418l28.512-91.997c.35-1.15-.09-1.858-1.33-1.858zm147.96.005c-43.653 0-60.654 37.719-60.654 62.157-.09 21.339 13.282 34.798 31.08 34.798v.004c11.868 0 21.693-5.314 29.133-16.117v12.836c0 1.061.62 1.596 1.594 1.596h14.166c.974 0 1.505-.446 1.86-1.42l28.777-92.086c.265-.973-.266-1.768-1.24-1.768zm-.616 20.54h17.265l-6.197 19.57c-7.35 23.289-18.684 37.896-32.585 37.896-10.094 0-15.585-6.907-15.585-18.15 0-17.976 13.722-39.315 37.102-39.315z"/><path fill="#850122" d="M226.561 106.964c-.558.007-1.043.428-1.043 1.095V251.59c0 1.594 2.04 1.594 2.48 0L261.38 143.3c.445-1.241.446-2.039-.62-3.1l-33.204-32.762c-.299-.332-.66-.478-.996-.474zm55.983 41.739c-1.241 0-1.594.444-2.039 1.417l-43.919 142.203c-.176.797.177 1.594 1.242 1.594h145.747c1.418 0 2.04-.62 2.48-1.858l44.098-141.499c.445-1.417-.18-1.857-1.417-1.857zm-40.022-58.62c-1.418 0-1.594 1.242-.797 2.04l35.065 35.24c.796.798 1.594 1.061 2.836 1.061h149.467c1.065 0 1.68-1.24.62-2.214l-34.63-34.885c-.796-.796-1.592-1.242-3.274-1.242z"/></svg>
|
||||||
|
After Width: | Height: | Size: 2.4 KiB |
202
_modules/index.html
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>Overview: module code — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">Overview: module code</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>All modules for which code is available</h1>
|
||||||
|
<ul><li><a href="builtins.html">builtins</a></li>
|
||||||
|
<li><a href="searx/answerers/_core.html">searx.answerers._core</a></li>
|
||||||
|
<li><a href="searx/answerers/random.html">searx.answerers.random</a></li>
|
||||||
|
<li><a href="searx/answerers/statistics.html">searx.answerers.statistics</a></li>
|
||||||
|
<li><a href="searx/autocomplete.html">searx.autocomplete</a></li>
|
||||||
|
<li><a href="searx/babel_extract.html">searx.babel_extract</a></li>
|
||||||
|
<li><a href="searx/botdetection/_helpers.html">searx.botdetection._helpers</a></li>
|
||||||
|
<li><a href="searx/botdetection/config.html">searx.botdetection.config</a></li>
|
||||||
|
<li><a href="searx/botdetection/http_sec_fetch.html">searx.botdetection.http_sec_fetch</a></li>
|
||||||
|
<li><a href="searx/botdetection/ip_lists.html">searx.botdetection.ip_lists</a></li>
|
||||||
|
<li><a href="searx/botdetection/link_token.html">searx.botdetection.link_token</a></li>
|
||||||
|
<li><a href="searx/botdetection/trusted_proxies.html">searx.botdetection.trusted_proxies</a></li>
|
||||||
|
<li><a href="searx/brand.html">searx.brand</a></li>
|
||||||
|
<li><a href="searx/cache.html">searx.cache</a></li>
|
||||||
|
<li><a href="searx/enginelib.html">searx.enginelib</a></li>
|
||||||
|
<ul><li><a href="searx/enginelib/traits.html">searx.enginelib.traits</a></li>
|
||||||
|
</ul><li><a href="searx/engines.html">searx.engines</a></li>
|
||||||
|
<ul><li><a href="searx/engines/annas_archive.html">searx.engines.annas_archive</a></li>
|
||||||
|
<li><a href="searx/engines/archlinux.html">searx.engines.archlinux</a></li>
|
||||||
|
<li><a href="searx/engines/astrophysics_data_system.html">searx.engines.astrophysics_data_system</a></li>
|
||||||
|
<li><a href="searx/engines/azure.html">searx.engines.azure</a></li>
|
||||||
|
<li><a href="searx/engines/bing.html">searx.engines.bing</a></li>
|
||||||
|
<li><a href="searx/engines/bing_images.html">searx.engines.bing_images</a></li>
|
||||||
|
<li><a href="searx/engines/bing_news.html">searx.engines.bing_news</a></li>
|
||||||
|
<li><a href="searx/engines/bing_videos.html">searx.engines.bing_videos</a></li>
|
||||||
|
<li><a href="searx/engines/brave.html">searx.engines.brave</a></li>
|
||||||
|
<li><a href="searx/engines/command.html">searx.engines.command</a></li>
|
||||||
|
<li><a href="searx/engines/core.html">searx.engines.core</a></li>
|
||||||
|
<li><a href="searx/engines/dailymotion.html">searx.engines.dailymotion</a></li>
|
||||||
|
<li><a href="searx/engines/demo_offline.html">searx.engines.demo_offline</a></li>
|
||||||
|
<li><a href="searx/engines/demo_online.html">searx.engines.demo_online</a></li>
|
||||||
|
<li><a href="searx/engines/duckduckgo.html">searx.engines.duckduckgo</a></li>
|
||||||
|
<li><a href="searx/engines/duckduckgo_definitions.html">searx.engines.duckduckgo_definitions</a></li>
|
||||||
|
<li><a href="searx/engines/github_code.html">searx.engines.github_code</a></li>
|
||||||
|
<li><a href="searx/engines/google.html">searx.engines.google</a></li>
|
||||||
|
<li><a href="searx/engines/google_images.html">searx.engines.google_images</a></li>
|
||||||
|
<li><a href="searx/engines/google_news.html">searx.engines.google_news</a></li>
|
||||||
|
<li><a href="searx/engines/google_scholar.html">searx.engines.google_scholar</a></li>
|
||||||
|
<li><a href="searx/engines/google_videos.html">searx.engines.google_videos</a></li>
|
||||||
|
<li><a href="searx/engines/json_engine.html">searx.engines.json_engine</a></li>
|
||||||
|
<li><a href="searx/engines/marginalia.html">searx.engines.marginalia</a></li>
|
||||||
|
<li><a href="searx/engines/mrs.html">searx.engines.mrs</a></li>
|
||||||
|
<li><a href="searx/engines/odysee.html">searx.engines.odysee</a></li>
|
||||||
|
<li><a href="searx/engines/peertube.html">searx.engines.peertube</a></li>
|
||||||
|
<li><a href="searx/engines/qwant.html">searx.engines.qwant</a></li>
|
||||||
|
<li><a href="searx/engines/radio_browser.html">searx.engines.radio_browser</a></li>
|
||||||
|
<li><a href="searx/engines/recoll.html">searx.engines.recoll</a></li>
|
||||||
|
<li><a href="searx/engines/reuters.html">searx.engines.reuters</a></li>
|
||||||
|
<li><a href="searx/engines/sepiasearch.html">searx.engines.sepiasearch</a></li>
|
||||||
|
<li><a href="searx/engines/springer.html">searx.engines.springer</a></li>
|
||||||
|
<li><a href="searx/engines/sqlite.html">searx.engines.sqlite</a></li>
|
||||||
|
<li><a href="searx/engines/startpage.html">searx.engines.startpage</a></li>
|
||||||
|
<li><a href="searx/engines/tineye.html">searx.engines.tineye</a></li>
|
||||||
|
<li><a href="searx/engines/torznab.html">searx.engines.torznab</a></li>
|
||||||
|
<li><a href="searx/engines/tubearchivist.html">searx.engines.tubearchivist</a></li>
|
||||||
|
<li><a href="searx/engines/voidlinux.html">searx.engines.voidlinux</a></li>
|
||||||
|
<li><a href="searx/engines/wikidata.html">searx.engines.wikidata</a></li>
|
||||||
|
<li><a href="searx/engines/wikipedia.html">searx.engines.wikipedia</a></li>
|
||||||
|
<li><a href="searx/engines/xpath.html">searx.engines.xpath</a></li>
|
||||||
|
<li><a href="searx/engines/yahoo.html">searx.engines.yahoo</a></li>
|
||||||
|
<li><a href="searx/engines/zlibrary.html">searx.engines.zlibrary</a></li>
|
||||||
|
</ul><li><a href="searx/exceptions.html">searx.exceptions</a></li>
|
||||||
|
<li><a href="searx/extended_types.html">searx.extended_types</a></li>
|
||||||
|
<li><a href="searx/favicons/cache.html">searx.favicons.cache</a></li>
|
||||||
|
<li><a href="searx/favicons/config.html">searx.favicons.config</a></li>
|
||||||
|
<li><a href="searx/favicons/proxy.html">searx.favicons.proxy</a></li>
|
||||||
|
<li><a href="searx/favicons/resolvers.html">searx.favicons.resolvers</a></li>
|
||||||
|
<li><a href="searx/infopage.html">searx.infopage</a></li>
|
||||||
|
<li><a href="searx/limiter.html">searx.limiter</a></li>
|
||||||
|
<li><a href="searx/locales.html">searx.locales</a></li>
|
||||||
|
<li><a href="searx/plugins/_core.html">searx.plugins._core</a></li>
|
||||||
|
<li><a href="searx/plugins/calculator.html">searx.plugins.calculator</a></li>
|
||||||
|
<li><a href="searx/plugins/hash_plugin.html">searx.plugins.hash_plugin</a></li>
|
||||||
|
<li><a href="searx/plugins/hostnames.html">searx.plugins.hostnames</a></li>
|
||||||
|
<li><a href="searx/plugins/infinite_scroll.html">searx.plugins.infinite_scroll</a></li>
|
||||||
|
<li><a href="searx/plugins/self_info.html">searx.plugins.self_info</a></li>
|
||||||
|
<li><a href="searx/plugins/time_zone.html">searx.plugins.time_zone</a></li>
|
||||||
|
<li><a href="searx/plugins/tor_check.html">searx.plugins.tor_check</a></li>
|
||||||
|
<li><a href="searx/plugins/unit_converter.html">searx.plugins.unit_converter</a></li>
|
||||||
|
<li><a href="searx/result_types.html">searx.result_types</a></li>
|
||||||
|
<ul><li><a href="searx/result_types/_base.html">searx.result_types._base</a></li>
|
||||||
|
<li><a href="searx/result_types/answer.html">searx.result_types.answer</a></li>
|
||||||
|
<li><a href="searx/result_types/code.html">searx.result_types.code</a></li>
|
||||||
|
<li><a href="searx/result_types/file.html">searx.result_types.file</a></li>
|
||||||
|
<li><a href="searx/result_types/keyvalue.html">searx.result_types.keyvalue</a></li>
|
||||||
|
<li><a href="searx/result_types/paper.html">searx.result_types.paper</a></li>
|
||||||
|
</ul><li><a href="searx/search.html">searx.search</a></li>
|
||||||
|
<ul><li><a href="searx/search/models.html">searx.search.models</a></li>
|
||||||
|
<li><a href="searx/search/processors/abstract.html">searx.search.processors.abstract</a></li>
|
||||||
|
<li><a href="searx/search/processors/offline.html">searx.search.processors.offline</a></li>
|
||||||
|
<li><a href="searx/search/processors/online.html">searx.search.processors.online</a></li>
|
||||||
|
<li><a href="searx/search/processors/online_currency.html">searx.search.processors.online_currency</a></li>
|
||||||
|
<li><a href="searx/search/processors/online_dictionary.html">searx.search.processors.online_dictionary</a></li>
|
||||||
|
<li><a href="searx/search/processors/online_url_search.html">searx.search.processors.online_url_search</a></li>
|
||||||
|
</ul><li><a href="searx/settings_loader.html">searx.settings_loader</a></li>
|
||||||
|
<li><a href="searx/sqlitedb.html">searx.sqlitedb</a></li>
|
||||||
|
<li><a href="searx/utils.html">searx.utils</a></li>
|
||||||
|
<li><a href="searx/valkeydb.html">searx.valkeydb</a></li>
|
||||||
|
<li><a href="searx/valkeylib.html">searx.valkeylib</a></li>
|
||||||
|
<li><a href="searx/weather.html">searx.weather</a></li>
|
||||||
|
<li><a href="searxng_extra/update/update_engine_descriptions.html">searxng_extra.update.update_engine_descriptions</a></li>
|
||||||
|
<li><a href="searxng_extra/update/update_engine_traits.html">searxng_extra.update.update_engine_traits</a></li>
|
||||||
|
<li><a href="searxng_extra/update/update_external_bangs.html">searxng_extra.update.update_external_bangs</a></li>
|
||||||
|
<li><a href="searxng_extra/update/update_locales.html">searxng_extra.update.update_locales</a></li>
|
||||||
|
<li><a href="searxng_extra/update/update_pygments.html">searxng_extra.update.update_pygments</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../index.html">
|
||||||
|
<img class="logo" src="../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Overview</a>
|
||||||
|
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
308
_modules/searx/answerers/_core.html
Normal file
@@ -0,0 +1,308 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.answerers._core — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.answerers._core</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.answerers._core</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods, missing-module-docstring</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">importlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">warnings</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">dataclasses</span><span class="w"> </span><span class="kn">import</span> <span class="n">dataclass</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_module</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types.answer</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseAnswer</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">_default</span> <span class="o">=</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span class="n">parent</span>
|
||||||
|
<span class="n">log</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"searx.answerers"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="AnswererInfo">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswererInfo">[docs]</a>
|
||||||
|
<span class="nd">@dataclass</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">AnswererInfo</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Object that holds information about an answerer, these infos are shown</span>
|
||||||
|
<span class="sd"> to the user in the Preferences menu.</span>
|
||||||
|
|
||||||
|
<span class="sd"> To be able to translate the information into other languages, the text must</span>
|
||||||
|
<span class="sd"> be written in English and translated with :py:obj:`flask_babel.gettext`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the *answerer*."""</span>
|
||||||
|
|
||||||
|
<span class="n">description</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Short description of the *answerer*."""</span>
|
||||||
|
|
||||||
|
<span class="n">examples</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of short examples of the usage / of query terms."""</span>
|
||||||
|
|
||||||
|
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""See :py:obj:`Answerer.keywords`"""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Answerer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.Answerer">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Answerer</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Abstract base class of answerers."""</span>
|
||||||
|
|
||||||
|
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Keywords to which the answerer has *answers*."""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Answerer.answer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.Answerer.answer">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">answer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Function that returns a list of answers to the question/query."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Answerer.info">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.Answerer.info">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">AnswererInfo</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Information about the *answerer*, see :py:obj:`AnswererInfo`."""</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ModuleAnswerer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.ModuleAnswerer">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ModuleAnswerer</span><span class="p">(</span><span class="n">Answerer</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A wrapper class for legacy *answerers* where the names (keywords, answer,</span>
|
||||||
|
<span class="sd"> info) are implemented on the module level (not in a class).</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> For internal use only!</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mod</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"keywords"</span><span class="p">,</span> <span class="s2">"self_info"</span><span class="p">,</span> <span class="s2">"answer"</span><span class="p">]:</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">SystemExit</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">mod</span><span class="o">.</span><span class="n">keywords</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">SystemExit</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">module</span> <span class="o">=</span> <span class="n">mod</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">keywords</span> <span class="o">=</span> <span class="n">mod</span><span class="o">.</span><span class="n">keywords</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ModuleAnswerer.answer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.ModuleAnswerer.answer">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">answer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="o">.</span><span class="n">answer</span><span class="p">(</span><span class="n">query</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ModuleAnswerer.info">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.ModuleAnswerer.info">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">AnswererInfo</span><span class="p">:</span>
|
||||||
|
<span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="o">.</span><span class="n">self_info</span><span class="p">()</span>
|
||||||
|
<span class="n">kwargs</span><span class="p">[</span><span class="s2">"keywords"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">keywords</span>
|
||||||
|
<span class="k">return</span> <span class="n">AnswererInfo</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="AnswerStorage">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">AnswerStorage</span><span class="p">(</span><span class="nb">dict</span><span class="p">):</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A storage for managing the *answerers* of SearXNG. With the</span>
|
||||||
|
<span class="sd"> :py:obj:`AnswerStorage.ask`” method, a caller can ask questions to all</span>
|
||||||
|
<span class="sd"> *answerers* and receives a list of the results."""</span>
|
||||||
|
|
||||||
|
<span class="n">answerer_list</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">Answerer</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The list of :py:obj:`Answerer` in this storage."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">answerer_list</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="AnswerStorage.load_builtins">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage.load_builtins">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">load_builtins</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Loads ``answerer.py`` modules from the python packages in</span>
|
||||||
|
<span class="sd"> :origin:`searx/answerers`. The python modules are wrapped by</span>
|
||||||
|
<span class="sd"> :py:obj:`ModuleAnswerer`."""</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">_default</span><span class="o">.</span><span class="n">iterdir</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"_"</span><span class="p">):</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">is_file</span><span class="p">()</span> <span class="ow">and</span> <span class="n">f</span><span class="o">.</span><span class="n">suffix</span> <span class="o">==</span> <span class="s2">".py"</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">register_by_fqn</span><span class="p">(</span><span class="sa">f</span><span class="s2">"searx.answerers.</span><span class="si">{</span><span class="n">f</span><span class="o">.</span><span class="n">stem</span><span class="si">}</span><span class="s2">.SXNGAnswerer"</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="c1"># for backward compatibility (if a fork has additional answerers)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">is_dir</span><span class="p">()</span> <span class="ow">and</span> <span class="p">(</span><span class="n">f</span> <span class="o">/</span> <span class="s2">"answerer.py"</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||||
|
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"answerer module </span><span class="si">{</span><span class="n">f</span><span class="si">}</span><span class="s2"> is deprecated / migrate to searx.answerers.Answerer"</span><span class="p">,</span> <span class="ne">DeprecationWarning</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">mod</span> <span class="o">=</span> <span class="n">load_module</span><span class="p">(</span><span class="s2">"answerer.py"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">f</span><span class="p">))</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">ModuleAnswerer</span><span class="p">(</span><span class="n">mod</span><span class="p">))</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="AnswerStorage.register_by_fqn">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage.register_by_fqn">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">register_by_fqn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fqn</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Register a :py:obj:`Answerer` via its fully qualified class namen(FQN)."""</span>
|
||||||
|
|
||||||
|
<span class="n">mod_name</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">obj_name</span> <span class="o">=</span> <span class="n">fqn</span><span class="o">.</span><span class="n">rpartition</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
|
||||||
|
<span class="n">mod</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">mod_name</span><span class="p">)</span>
|
||||||
|
<span class="n">code_obj</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">obj_name</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">code_obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"answerer </span><span class="si">{</span><span class="n">fqn</span><span class="si">}</span><span class="s2"> is not implemented"</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">code_obj</span><span class="p">())</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="AnswerStorage.register">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage.register">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">answerer</span><span class="p">:</span> <span class="n">Answerer</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Register a :py:obj:`Answerer`."""</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">answerer_list</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">answerer</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">_kw</span> <span class="ow">in</span> <span class="n">answerer</span><span class="o">.</span><span class="n">keywords</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="n">_kw</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">_kw</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="n">_kw</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">answerer</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="AnswerStorage.ask">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/development.html#searx.answerers.AnswerStorage.ask">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">ask</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""An answerer is identified via keywords, if there is a keyword at the</span>
|
||||||
|
<span class="sd"> first position in the ``query`` for which there is one or more</span>
|
||||||
|
<span class="sd"> answerers, then these are called, whereby the entire ``query`` is passed</span>
|
||||||
|
<span class="sd"> as argument to the answerer function."""</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">keyword</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">for</span> <span class="n">keyword</span> <span class="ow">in</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="n">keyword</span><span class="p">:</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">keyword</span> <span class="ow">or</span> <span class="n">keyword</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">answerer</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">[</span><span class="n">keyword</span><span class="p">]:</span>
|
||||||
|
<span class="k">for</span> <span class="n">answer</span> <span class="ow">in</span> <span class="n">answerer</span><span class="o">.</span><span class="n">answer</span><span class="p">(</span><span class="n">query</span><span class="p">):</span>
|
||||||
|
<span class="c1"># In case of *answers* prefix ``answerer:`` is set, see searx.result_types.Result</span>
|
||||||
|
<span class="n">answer</span><span class="o">.</span><span class="n">engine</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"answerer: </span><span class="si">{</span><span class="n">keyword</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">answer</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="nd">@property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">AnswererInfo</span><span class="p">]:</span>
|
||||||
|
<span class="k">return</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">info</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">answerer_list</span><span class="p">]</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
192
_modules/searx/answerers/random.html
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.answerers.random — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.answerers.random</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.answerers.random</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">uuid</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">Answer</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types.answer</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseAnswer</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Answerer</span><span class="p">,</span> <span class="n">AnswererInfo</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">random_characters</span><span class="p">():</span>
|
||||||
|
<span class="n">random_string_letters</span> <span class="o">=</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_lowercase</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_uppercase</span>
|
||||||
|
<span class="k">return</span> <span class="p">[</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">random_string_letters</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="mi">32</span><span class="p">))]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">random_string</span><span class="p">():</span>
|
||||||
|
<span class="k">return</span> <span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">random_characters</span><span class="p">())</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">random_float</span><span class="p">():</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">())</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">random_int</span><span class="p">():</span>
|
||||||
|
<span class="n">random_int_max</span> <span class="o">=</span> <span class="mi">2</span><span class="o">**</span><span class="mi">31</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="o">-</span><span class="n">random_int_max</span><span class="p">,</span> <span class="n">random_int_max</span><span class="p">))</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">random_sha256</span><span class="p">():</span>
|
||||||
|
<span class="n">m</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">()</span>
|
||||||
|
<span class="n">m</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">random_characters</span><span class="p">())</span><span class="o">.</span><span class="n">encode</span><span class="p">())</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">())</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">random_uuid</span><span class="p">():</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">uuid</span><span class="o">.</span><span class="n">uuid4</span><span class="p">())</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">random_color</span><span class="p">():</span>
|
||||||
|
<span class="n">color</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%06x</span><span class="s2">"</span> <span class="o">%</span> <span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mh">0xFFFFFF</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"#</span><span class="si">{</span><span class="n">color</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGAnswerer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/random.html#searx.answerers.random.SXNGAnswerer">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGAnswerer</span><span class="p">(</span><span class="n">Answerer</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Random value generator"""</span>
|
||||||
|
|
||||||
|
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"random"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">random_types</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"string"</span><span class="p">:</span> <span class="n">random_string</span><span class="p">,</span>
|
||||||
|
<span class="s2">"int"</span><span class="p">:</span> <span class="n">random_int</span><span class="p">,</span>
|
||||||
|
<span class="s2">"float"</span><span class="p">:</span> <span class="n">random_float</span><span class="p">,</span>
|
||||||
|
<span class="s2">"sha256"</span><span class="p">:</span> <span class="n">random_sha256</span><span class="p">,</span>
|
||||||
|
<span class="s2">"uuid"</span><span class="p">:</span> <span class="n">random_uuid</span><span class="p">,</span>
|
||||||
|
<span class="s2">"color"</span><span class="p">:</span> <span class="n">random_color</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGAnswerer.info">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/random.html#searx.answerers.random.SXNGAnswerer.info">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">AnswererInfo</span><span class="p">(</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__doc__</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Generate different random values"</span><span class="p">),</span>
|
||||||
|
<span class="n">keywords</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">,</span>
|
||||||
|
<span class="n">examples</span><span class="o">=</span><span class="p">[</span><span class="sa">f</span><span class="s2">"random </span><span class="si">{</span><span class="n">x</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">random_types</span><span class="p">],</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGAnswerer.answer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/random.html#searx.answerers.random.SXNGAnswerer.answer">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">answer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="n">parts</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">parts</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">2</span> <span class="ow">or</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">random_types</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="p">[</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">random_types</span><span class="p">[</span><span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]]())]</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
178
_modules/searx/answerers/statistics.html
Normal file
@@ -0,0 +1,178 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.answerers.statistics — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.answerers.statistics</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.answerers.statistics</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">functools</span><span class="w"> </span><span class="kn">import</span> <span class="n">reduce</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">operator</span><span class="w"> </span><span class="kn">import</span> <span class="n">mul</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.numbers</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">sxng_request</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">Answer</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types.answer</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseAnswer</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Answerer</span><span class="p">,</span> <span class="n">AnswererInfo</span>
|
||||||
|
|
||||||
|
<span class="n">kw2func</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="p">(</span><span class="s2">"min"</span><span class="p">,</span> <span class="nb">min</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"max"</span><span class="p">,</span> <span class="nb">max</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"avg"</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="nb">sum</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o">/</span> <span class="nb">len</span><span class="p">(</span><span class="n">args</span><span class="p">)),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"sum"</span><span class="p">,</span> <span class="nb">sum</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"prod"</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="n">reduce</span><span class="p">(</span><span class="n">mul</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="mi">1</span><span class="p">)),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"range"</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">args</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o">-</span> <span class="nb">min</span><span class="p">(</span><span class="n">args</span><span class="p">)),</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGAnswerer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/statistics.html#searx.answerers.statistics.SXNGAnswerer">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGAnswerer</span><span class="p">(</span><span class="n">Answerer</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Statistics functions"""</span>
|
||||||
|
|
||||||
|
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="n">kw</span> <span class="k">for</span> <span class="n">kw</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">kw2func</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGAnswerer.info">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/statistics.html#searx.answerers.statistics.SXNGAnswerer.info">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">AnswererInfo</span><span class="p">(</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__doc__</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Compute </span><span class="si">{func}</span><span class="s2"> of the arguments"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">func</span><span class="o">=</span><span class="s1">'/'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">))),</span>
|
||||||
|
<span class="n">keywords</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">,</span>
|
||||||
|
<span class="n">examples</span><span class="o">=</span><span class="p">[</span><span class="s2">"avg 123 548 2.04 24.2"</span><span class="p">],</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGAnswerer.answer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/answerers/statistics.html#searx.answerers.statistics.SXNGAnswerer.answer">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">answer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">BaseAnswer</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">parts</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">parts</span><span class="p">)</span> <span class="o"><</span> <span class="mi">2</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">ui_locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">'locale'</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">[</span><span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">parse_decimal</span><span class="p">(</span><span class="n">num</span><span class="p">,</span> <span class="n">ui_locale</span><span class="p">,</span> <span class="n">numbering_system</span><span class="o">=</span><span class="s2">"latn"</span><span class="p">)</span> <span class="k">for</span> <span class="n">num</span> <span class="ow">in</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">:]]</span>
|
||||||
|
<span class="k">except</span><span class="p">:</span> <span class="c1"># pylint: disable=bare-except</span>
|
||||||
|
<span class="c1"># seems one of the args is not a float type, can't be converted to float</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">func</span> <span class="ow">in</span> <span class="n">kw2func</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">k</span> <span class="o">==</span> <span class="n">parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">func</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">format_decimal</span><span class="p">(</span><span class="n">res</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">ui_locale</span><span class="p">)</span>
|
||||||
|
<span class="n">f_str</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">format_decimal</span><span class="p">(</span><span class="n">arg</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">ui_locale</span><span class="p">)</span> <span class="k">for</span> <span class="n">arg</span> <span class="ow">in</span> <span class="n">args</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="sa">f</span><span class="s2">"[</span><span class="si">{</span><span class="n">ui_locale</span><span class="si">}</span><span class="s2">] </span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">(</span><span class="si">{</span><span class="n">f_str</span><span class="si">}</span><span class="s2">) = </span><span class="si">{</span><span class="n">res</span><span class="si">}</span><span class="s2"> "</span><span class="p">))</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
482
_modules/searx/autocomplete.html
Normal file
@@ -0,0 +1,482 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.autocomplete — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.autocomplete</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.autocomplete</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This module implements functions needed for the autocompleter."""</span>
|
||||||
|
<span class="c1"># pylint: disable=use-dict-literal</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">html</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">quote_plus</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.etree</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.html</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">httpx</span><span class="w"> </span><span class="kn">import</span> <span class="n">HTTPError</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">settings</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">engines</span><span class="p">,</span>
|
||||||
|
<span class="n">google</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="k">as</span> <span class="n">http_get</span><span class="p">,</span> <span class="n">post</span> <span class="k">as</span> <span class="n">http_post</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineResponseException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extr</span><span class="p">,</span> <span class="n">gen_useragent</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">update_kwargs</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'timeout'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
|
||||||
|
<span class="n">kwargs</span><span class="p">[</span><span class="s1">'timeout'</span><span class="p">]</span> <span class="o">=</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">][</span><span class="s1">'request_timeout'</span><span class="p">]</span>
|
||||||
|
<span class="n">kwargs</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"SXNG_Response"</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">update_kwargs</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
<span class="k">return</span> <span class="n">http_get</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">post</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"SXNG_Response"</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">update_kwargs</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
<span class="k">return</span> <span class="n">http_post</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">baidu</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># baidu search autocompleter</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://www.baidu.com/sugrec?"</span>
|
||||||
|
<span class="n">response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">base_url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'ie'</span><span class="p">:</span> <span class="s1">'utf-8'</span><span class="p">,</span> <span class="s1">'json'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'prod'</span><span class="p">:</span> <span class="s1">'pc'</span><span class="p">,</span> <span class="s1">'wd'</span><span class="p">:</span> <span class="n">query</span><span class="p">}))</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'g'</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s1">'g'</span><span class="p">]:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">'q'</span><span class="p">])</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">brave</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># brave search autocompleter</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s1">'https://search.brave.com/api/suggest?'</span>
|
||||||
|
<span class="n">url</span> <span class="o">+=</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">})</span>
|
||||||
|
<span class="n">country</span> <span class="o">=</span> <span class="s1">'all'</span>
|
||||||
|
<span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'cookies'</span><span class="p">:</span> <span class="p">{</span><span class="s1">'country'</span><span class="p">:</span> <span class="n">country</span><span class="p">}}</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">]:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">dbpedia</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="n">autocomplete_url</span> <span class="o">=</span> <span class="s1">'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">autocomplete_url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">QueryString</span><span class="o">=</span><span class="n">query</span><span class="p">)))</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">etree</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//Result/Label//text()'</span><span class="p">)]</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">duckduckgo</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Autocomplete from DuckDuckGo. Supports DuckDuckGo's languages"""</span>
|
||||||
|
|
||||||
|
<span class="n">traits</span> <span class="o">=</span> <span class="n">engines</span><span class="p">[</span><span class="s1">'duckduckgo'</span><span class="p">]</span><span class="o">.</span><span class="n">traits</span>
|
||||||
|
<span class="n">args</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'kl'</span><span class="p">:</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s1">'https://duckduckgo.com/ac/?type=list&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">j</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">j</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">j</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="google_complete">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/online/google.html#searx.autocomplete.google_complete">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">google_complete</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Autocomplete from Google. Supports Google's languages and subdomains</span>
|
||||||
|
<span class="sd"> (:py:obj:`searx.engines.google.get_google_info`) by using the async REST</span>
|
||||||
|
<span class="sd"> API::</span>
|
||||||
|
|
||||||
|
<span class="sd"> https://{subdomain}/complete/search?{args}</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">google_info</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">google</span><span class="o">.</span><span class="n">get_google_info</span><span class="p">({</span><span class="s1">'searxng_locale'</span><span class="p">:</span> <span class="n">sxng_locale</span><span class="p">},</span> <span class="n">engines</span><span class="p">[</span><span class="s1">'google'</span><span class="p">]</span><span class="o">.</span><span class="n">traits</span><span class="p">)</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s1">'https://</span><span class="si">{subdomain}</span><span class="s1">/complete/search?</span><span class="si">{args}</span><span class="s1">'</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'client'</span><span class="p">:</span> <span class="s1">'gws-wiz'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'hl'</span><span class="p">:</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subdomain</span><span class="o">=</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">],</span> <span class="n">args</span><span class="o">=</span><span class="n">args</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">resp</span> <span class="ow">and</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">json_txt</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">[</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'['</span><span class="p">)</span> <span class="p">:</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">']'</span><span class="p">,</span> <span class="o">-</span><span class="mi">3</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">json_txt</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">text_content</span><span class="p">())</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="mwmbl">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/online/mwmbl.html#searx.autocomplete.mwmbl">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">mwmbl</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Autocomplete from Mwmbl_."""</span>
|
||||||
|
|
||||||
|
<span class="c1"># mwmbl autocompleter</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s1">'https://api.mwmbl.org/search/complete?</span><span class="si">{query}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">})))</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># results starting with `go:` are direct urls and not useful for auto completion</span>
|
||||||
|
<span class="k">return</span> <span class="p">[</span><span class="n">result</span> <span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">results</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">result</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"go: "</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">result</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"search: "</span><span class="p">)]</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">naver</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># Naver search autocompleter</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://ac.search.naver.com/nx/ac?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span><span class="w"> </span><span class="n">query</span><span class="p">,</span><span class="w"> </span><span class="s1">'r_format'</span><span class="p">:</span><span class="w"> </span><span class="s1">'json'</span><span class="p">,</span><span class="w"> </span><span class="s1">'st'</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">})</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'items'</span><span class="p">):</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s1">'items'</span><span class="p">][</span><span class="mi">0</span><span class="p">]:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">qihu360search</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># 360Search search autocompleter</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://sug.so.360.cn/suggest?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'format'</span><span class="p">:</span><span class="w"> </span><span class="s1">'json'</span><span class="p">,</span><span class="w"> </span><span class="s1">'word'</span><span class="p">:</span><span class="w"> </span><span class="n">query</span><span class="p">})</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'result'</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s1">'result'</span><span class="p">]:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">'word'</span><span class="p">])</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">quark</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># Quark search autocompleter</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://sugs.m.sm.cn/web?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span><span class="w"> </span><span class="n">query</span><span class="p">})</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'r'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">'w'</span><span class="p">])</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">seznam</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># seznam search autocompleter</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s1">'https://suggest.seznam.cz/fulltext/cs?</span><span class="si">{query}</span><span class="s1">'</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span>
|
||||||
|
<span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||||
|
<span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span><span class="s1">'phrase'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span> <span class="s1">'cursorPosition'</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">query</span><span class="p">),</span> <span class="s1">'format'</span><span class="p">:</span> <span class="s1">'json-2'</span><span class="p">,</span> <span class="s1">'highlight'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span> <span class="s1">'count'</span><span class="p">:</span> <span class="s1">'6'</span><span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">part</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'text'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'text'</span><span class="p">,</span> <span class="p">[])])</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'result'</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="k">if</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'itemType'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="o">==</span> <span class="s1">'ItemType.TEXT'</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">sogou</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># Sogou search autocompleter</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://sor.html5.qq.com/api/getsug?"</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">base_url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'m'</span><span class="p">:</span> <span class="s1">'searxng'</span><span class="p">,</span> <span class="s1">'key'</span><span class="p">:</span> <span class="n">query</span><span class="p">}))</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">raw_json</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s2">"["</span><span class="p">,</span> <span class="s2">"]"</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="sa">f</span><span class="s2">"[</span><span class="si">{</span><span class="n">raw_json</span><span class="si">}</span><span class="s2">]]"</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">except</span> <span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">:</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">startpage</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Autocomplete from Startpage's Firefox extension.</span>
|
||||||
|
<span class="sd"> Supports the languages specified in lang_map.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">lang_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'da'</span><span class="p">:</span> <span class="s1">'dansk'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'de'</span><span class="p">:</span> <span class="s1">'deutsch'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'en'</span><span class="p">:</span> <span class="s1">'english'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es'</span><span class="p">:</span> <span class="s1">'espanol'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'fr'</span><span class="p">:</span> <span class="s1">'francais'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'nb'</span><span class="p">:</span> <span class="s1">'norsk'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'nl'</span><span class="p">:</span> <span class="s1">'nederlands'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'pl'</span><span class="p">:</span> <span class="s1">'polski'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'pt'</span><span class="p">:</span> <span class="s1">'portugues'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'sv'</span><span class="p">:</span> <span class="s1">'svenska'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">base_lang</span> <span class="o">=</span> <span class="n">sxng_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">lui</span> <span class="o">=</span> <span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">base_lang</span><span class="p">,</span> <span class="s1">'english'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">url_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'format'</span><span class="p">:</span> <span class="s1">'opensearch'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'segment'</span><span class="p">:</span> <span class="s1">'startpage.defaultffx'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'lui'</span><span class="p">:</span> <span class="n">lui</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'https://www.startpage.com/suggestions?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">url_params</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="c1"># Needs user agent, returns a 204 otherwise</span>
|
||||||
|
<span class="n">h</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'User-Agent'</span><span class="p">:</span> <span class="n">gen_useragent</span><span class="p">()}</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">h</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">2</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">except</span> <span class="n">json</span><span class="o">.</span><span class="n">JSONDecodeError</span><span class="p">:</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">stract</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># stract autocompleter (beta)</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://stract.com/beta/api/autosuggest?q=</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">query</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">post</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[</span><span class="n">html</span><span class="o">.</span><span class="n">unescape</span><span class="p">(</span><span class="n">suggestion</span><span class="p">[</span><span class="s1">'raw'</span><span class="p">])</span> <span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()]</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">swisscows</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># swisscows autocompleter</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s1">'https://swisscows.ch/api/suggest?</span><span class="si">{query}</span><span class="s1">&itemsCount=5'</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">})))</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">qwant</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Autocomplete from Qwant. Supports Qwant's regions."""</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">engines</span><span class="p">[</span><span class="s1">'qwant'</span><span class="p">]</span><span class="o">.</span><span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="s1">'en_US'</span><span class="p">)</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s1">'https://api.qwant.com/v3/suggest?</span><span class="si">{query}</span><span class="s1">'</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span> <span class="s1">'locale'</span><span class="p">:</span> <span class="n">locale</span><span class="p">,</span> <span class="s1">'version'</span><span class="p">:</span> <span class="s1">'2'</span><span class="p">})))</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">data</span><span class="p">[</span><span class="s1">'status'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'success'</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'items'</span><span class="p">]:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">[</span><span class="s1">'value'</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">wikipedia</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Autocomplete from Wikipedia. Supports Wikipedia's languages (aka netloc)."""</span>
|
||||||
|
<span class="n">eng_traits</span> <span class="o">=</span> <span class="n">engines</span><span class="p">[</span><span class="s1">'wikipedia'</span><span class="p">]</span><span class="o">.</span><span class="n">traits</span>
|
||||||
|
<span class="n">wiki_lang</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="s1">'en'</span><span class="p">)</span>
|
||||||
|
<span class="n">wiki_netloc</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">wiki_lang</span><span class="p">,</span> <span class="s1">'en.wikipedia.org'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'action'</span><span class="p">:</span> <span class="s1">'opensearch'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'format'</span><span class="p">:</span> <span class="s1">'json'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'formatversion'</span><span class="p">:</span> <span class="s1">'2'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'search'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'namespace'</span><span class="p">:</span> <span class="s1">'0'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'limit'</span><span class="p">:</span> <span class="s1">'10'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s1">'https://</span><span class="si">{</span><span class="n">wiki_netloc</span><span class="si">}</span><span class="s1">/w/api.php?</span><span class="si">{</span><span class="n">args</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">yandex</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">_sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># yandex autocompleter</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s2">"https://suggest.yandex.com/suggest-ff.cgi?</span><span class="si">{0}</span><span class="s2">"</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">urlencode</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">part</span><span class="o">=</span><span class="n">query</span><span class="p">))))</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">resp</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">backends</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'360search'</span><span class="p">:</span> <span class="n">qihu360search</span><span class="p">,</span>
|
||||||
|
<span class="s1">'baidu'</span><span class="p">:</span> <span class="n">baidu</span><span class="p">,</span>
|
||||||
|
<span class="s1">'brave'</span><span class="p">:</span> <span class="n">brave</span><span class="p">,</span>
|
||||||
|
<span class="s1">'dbpedia'</span><span class="p">:</span> <span class="n">dbpedia</span><span class="p">,</span>
|
||||||
|
<span class="s1">'duckduckgo'</span><span class="p">:</span> <span class="n">duckduckgo</span><span class="p">,</span>
|
||||||
|
<span class="s1">'google'</span><span class="p">:</span> <span class="n">google_complete</span><span class="p">,</span>
|
||||||
|
<span class="s1">'mwmbl'</span><span class="p">:</span> <span class="n">mwmbl</span><span class="p">,</span>
|
||||||
|
<span class="s1">'naver'</span><span class="p">:</span> <span class="n">naver</span><span class="p">,</span>
|
||||||
|
<span class="s1">'quark'</span><span class="p">:</span> <span class="n">quark</span><span class="p">,</span>
|
||||||
|
<span class="s1">'qwant'</span><span class="p">:</span> <span class="n">qwant</span><span class="p">,</span>
|
||||||
|
<span class="s1">'seznam'</span><span class="p">:</span> <span class="n">seznam</span><span class="p">,</span>
|
||||||
|
<span class="s1">'sogou'</span><span class="p">:</span> <span class="n">sogou</span><span class="p">,</span>
|
||||||
|
<span class="s1">'startpage'</span><span class="p">:</span> <span class="n">startpage</span><span class="p">,</span>
|
||||||
|
<span class="s1">'stract'</span><span class="p">:</span> <span class="n">stract</span><span class="p">,</span>
|
||||||
|
<span class="s1">'swisscows'</span><span class="p">:</span> <span class="n">swisscows</span><span class="p">,</span>
|
||||||
|
<span class="s1">'wikipedia'</span><span class="p">:</span> <span class="n">wikipedia</span><span class="p">,</span>
|
||||||
|
<span class="s1">'yandex'</span><span class="p">:</span> <span class="n">yandex</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">search_autocomplete</span><span class="p">(</span><span class="n">backend_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="n">backend</span> <span class="o">=</span> <span class="n">backends</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">backend_name</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">backend</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="p">[]</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">backend</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="p">(</span><span class="n">HTTPError</span><span class="p">,</span> <span class="n">SearxEngineResponseException</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="p">[]</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
165
_modules/searx/babel_extract.html
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.babel_extract — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.babel_extract</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.babel_extract</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This module implements the :origin:`searxng_msg <babel.cfg>` extractor to</span>
|
||||||
|
<span class="sd">extract messages from:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :origin:`searx/searxng.msg`</span>
|
||||||
|
|
||||||
|
<span class="sd">The ``searxng.msg`` files are selected by Babel_, see Babel's configuration in</span>
|
||||||
|
<span class="sd">:origin:`babel.cfg`::</span>
|
||||||
|
|
||||||
|
<span class="sd"> searxng_msg = searx.babel_extract.extract</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> [searxng_msg: **/searxng.msg]</span>
|
||||||
|
|
||||||
|
<span class="sd">A ``searxng.msg`` file is a python file that is *executed* by the</span>
|
||||||
|
<span class="sd">:py:obj:`extract` function. Additional ``searxng.msg`` files can be added by:</span>
|
||||||
|
|
||||||
|
<span class="sd">1. Adding a ``searxng.msg`` file in one of the SearXNG python packages and</span>
|
||||||
|
<span class="sd">2. implement a method in :py:obj:`extract` that yields messages from this file.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Babel: https://babel.pocoo.org/en/latest/index.html</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">os</span><span class="w"> </span><span class="kn">import</span> <span class="n">path</span>
|
||||||
|
|
||||||
|
<span class="n">SEARXNG_MSG_FILE</span> <span class="o">=</span> <span class="s2">"searxng.msg"</span>
|
||||||
|
<span class="n">_MSG_FILES</span> <span class="o">=</span> <span class="p">[</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">),</span> <span class="n">SEARXNG_MSG_FILE</span><span class="p">)]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="extract">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.babel_extract.html#searx.babel_extract.extract">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">extract</span><span class="p">(</span>
|
||||||
|
<span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="n">fileobj</span><span class="p">,</span>
|
||||||
|
<span class="n">keywords</span><span class="p">,</span>
|
||||||
|
<span class="n">comment_tags</span><span class="p">,</span>
|
||||||
|
<span class="n">options</span><span class="p">,</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Extract messages from ``searxng.msg`` files by a custom extractor_.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _extractor:</span>
|
||||||
|
<span class="sd"> https://babel.pocoo.org/en/latest/messages.html#writing-extraction-methods</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="n">fileobj</span><span class="o">.</span><span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_MSG_FILES</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"don't know how to extract messages from </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">fileobj</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">namespace</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">exec</span><span class="p">(</span><span class="n">fileobj</span><span class="o">.</span><span class="n">read</span><span class="p">(),</span> <span class="p">{},</span> <span class="n">namespace</span><span class="p">)</span> <span class="c1"># pylint: disable=exec-used</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">obj_name</span> <span class="ow">in</span> <span class="n">namespace</span><span class="p">[</span><span class="s1">'__all__'</span><span class="p">]:</span>
|
||||||
|
<span class="n">obj</span> <span class="o">=</span> <span class="n">namespace</span><span class="p">[</span><span class="n">obj_name</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||||
|
<span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">:</span>
|
||||||
|
<span class="c1"># (lineno, funcname, message, comments)</span>
|
||||||
|
<span class="k">yield</span> <span class="mi">0</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="p">[</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">obj_name</span><span class="si">}</span><span class="s2">"</span><span class="p">]</span>
|
||||||
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">yield</span> <span class="mi">0</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">,</span> <span class="n">msg</span><span class="p">,</span> <span class="p">[</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">obj_name</span><span class="si">}</span><span class="s2">['</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">']"</span><span class="p">]</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">obj_name</span><span class="si">}</span><span class="s2"> should be list or dict"</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
196
_modules/searx/botdetection/_helpers.html
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.botdetection._helpers — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.botdetection._helpers</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.botdetection._helpers</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring, invalid-name</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"log_error_only_once"</span><span class="p">,</span> <span class="s2">"dump_request"</span><span class="p">,</span> <span class="s2">"get_network"</span><span class="p">,</span> <span class="s2">"logger"</span><span class="p">,</span> <span class="s2">"too_many_requests"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">IPv4Network</span><span class="p">,</span>
|
||||||
|
<span class="n">IPv6Network</span><span class="p">,</span>
|
||||||
|
<span class="n">IPv4Address</span><span class="p">,</span>
|
||||||
|
<span class="n">IPv6Address</span><span class="p">,</span>
|
||||||
|
<span class="n">ip_network</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">werkzeug</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'botdetection'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">dump_request</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="p">(</span>
|
||||||
|
<span class="n">request</span><span class="o">.</span><span class="n">path</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || X-Forwarded-For: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'X-Forwarded-For'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || X-Real-IP: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'X-Real-IP'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || form: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">form</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Accept: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Accept'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Accept-Language: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Accept-Language'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Accept-Encoding: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Accept-Encoding'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Content-Type: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Content-Type'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Content-Length: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Content-Length'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Connection: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Connection'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || User-Agent: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'User-Agent'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Sec-Fetch-Site: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Sec-Fetch-Site'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Sec-Fetch-Mode: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Sec-Fetch-Mode'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">" || Sec-Fetch-Dest: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Sec-Fetch-Dest'</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="too_many_requests">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.too_many_requests">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">too_many_requests</span><span class="p">(</span><span class="n">network</span><span class="p">:</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">,</span> <span class="n">log_msg</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">werkzeug</span><span class="o">.</span><span class="n">Response</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a HTTP 429 response object and writes a ERROR message to the</span>
|
||||||
|
<span class="sd"> 'botdetection' logger. This function is used in part by the filter methods</span>
|
||||||
|
<span class="sd"> to return the default ``Too Many Requests`` response.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"BLOCK </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">log_msg</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">make_response</span><span class="p">((</span><span class="s1">'Too Many Requests'</span><span class="p">,</span> <span class="mi">429</span><span class="p">))</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_network">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.get_network">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_network</span><span class="p">(</span><span class="n">real_ip</span><span class="p">:</span> <span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="s2">"config.Config"</span><span class="p">)</span> <span class="o">-></span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns the (client) network of whether the ``real_ip`` is part of.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The ``ipv4_prefix`` and ``ipv6_prefix`` define the number of leading bits in</span>
|
||||||
|
<span class="sd"> an address that are compared to determine whether or not an address is part</span>
|
||||||
|
<span class="sd"> of a (client) network.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: toml</span>
|
||||||
|
|
||||||
|
<span class="sd"> [botdetection]</span>
|
||||||
|
|
||||||
|
<span class="sd"> ipv4_prefix = 32</span>
|
||||||
|
<span class="sd"> ipv6_prefix = 48</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">prefix</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">cfg</span><span class="p">[</span><span class="s2">"botdetection.ipv4_prefix"</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">real_ip</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="mi">6</span><span class="p">:</span>
|
||||||
|
<span class="n">prefix</span> <span class="o">=</span> <span class="n">cfg</span><span class="p">[</span><span class="s2">"botdetection.ipv6_prefix"</span><span class="p">]</span>
|
||||||
|
<span class="n">network</span> <span class="o">=</span> <span class="n">ip_network</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">real_ip</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">prefix</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="c1"># logger.debug("get_network(): %s", network.compressed)</span>
|
||||||
|
<span class="k">return</span> <span class="n">network</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">_logged_errors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">log_error_only_once</span><span class="p">(</span><span class="n">err_msg</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">err_msg</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_logged_errors</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
|
||||||
|
<span class="n">_logged_errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
528
_modules/searx/botdetection/config.html
Normal file
@@ -0,0 +1,528 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.botdetection.config — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.botdetection.config</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.botdetection.config</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Configuration class :py:class:`Config` with deep-update, schema validation</span>
|
||||||
|
<span class="sd">and deprecated names.</span>
|
||||||
|
|
||||||
|
<span class="sd">The :py:class:`Config` class implements a configuration that is based on</span>
|
||||||
|
<span class="sd">structured dictionaries. The configuration schema is defined in a dictionary</span>
|
||||||
|
<span class="sd">structure and the configuration data is given in a dictionary structure.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">copy</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">..compat</span><span class="w"> </span><span class="kn">import</span> <span class="n">tomllib</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'Config'</span><span class="p">,</span> <span class="s1">'UNSET'</span><span class="p">,</span> <span class="s1">'SchemaIssue'</span><span class="p">,</span> <span class="s1">'set_global_cfg'</span><span class="p">,</span> <span class="s1">'get_global_cfg'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">CFG</span><span class="p">:</span> <span class="s2">"Config | None"</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">"""Global config of the botdetection."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set_global_cfg</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="s2">"Config"</span><span class="p">):</span>
|
||||||
|
<span class="k">global</span> <span class="n">CFG</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
<span class="n">CFG</span> <span class="o">=</span> <span class="n">cfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_global_cfg</span><span class="p">()</span> <span class="o">-></span> <span class="s2">"Config"</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">CFG</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Botdetection's config is not yet initialized."</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">CFG</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="nd">@typing</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FALSE</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Class of ``False`` singleton"""</span>
|
||||||
|
|
||||||
|
<span class="c1"># pylint: disable=multiple-statements</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">msg</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">msg</span> <span class="o">=</span> <span class="n">msg</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__bool__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">msg</span>
|
||||||
|
|
||||||
|
<span class="fm">__repr__</span> <span class="o">=</span> <span class="fm">__str__</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">UNSET</span> <span class="o">=</span> <span class="n">FALSE</span><span class="p">(</span><span class="s1">'<UNSET>'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SchemaIssue">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.SchemaIssue">[docs]</a>
|
||||||
|
<span class="nd">@typing</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SchemaIssue</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Exception to store and/or raise a message from a schema issue."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">level</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s1">'warn'</span><span class="p">,</span> <span class="s1">'invalid'</span><span class="p">],</span> <span class="n">msg</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">level</span> <span class="o">=</span> <span class="n">level</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"[cfg schema </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">level</span><span class="si">}</span><span class="s2">] </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Config">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Config</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Base class used for configuration"""</span>
|
||||||
|
|
||||||
|
<span class="n">UNSET</span><span class="p">:</span> <span class="nb">object</span> <span class="o">=</span> <span class="n">UNSET</span>
|
||||||
|
|
||||||
|
<span class="nd">@classmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">from_toml</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">schema_file</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">,</span> <span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="s2">"Config"</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="c1"># init schema</span>
|
||||||
|
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"load schema file: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">schema_file</span><span class="p">)</span>
|
||||||
|
<span class="n">cfg</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">cfg_schema</span><span class="o">=</span><span class="n">toml_load</span><span class="p">(</span><span class="n">schema_file</span><span class="p">),</span> <span class="n">deprecated</span><span class="o">=</span><span class="n">deprecated</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">cfg_file</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"missing config file: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">cfg</span>
|
||||||
|
|
||||||
|
<span class="c1"># load configuration</span>
|
||||||
|
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"load config file: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">)</span>
|
||||||
|
<span class="n">upd_cfg</span> <span class="o">=</span> <span class="n">toml_load</span><span class="p">(</span><span class="n">cfg_file</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">is_valid</span><span class="p">,</span> <span class="n">issue_list</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">validate</span><span class="p">(</span><span class="n">upd_cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">issue_list</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">msg</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_valid</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"schema of </span><span class="si">{</span><span class="n">cfg_file</span><span class="si">}</span><span class="s2"> is invalid!"</span><span class="p">)</span>
|
||||||
|
<span class="n">cfg</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">upd_cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">cfg</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg_schema</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Constructor of class Config.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param cfg_schema: Schema of the configuration</span>
|
||||||
|
<span class="sd"> :param deprecated: dictionary that maps deprecated configuration names to a messages</span>
|
||||||
|
|
||||||
|
<span class="sd"> These values are needed for validation, see :py:obj:`validate`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">cfg_schema</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">cfg_schema</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">deprecated</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">cfg_schema</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Config.validate">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.validate">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">validate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Validation of dictionary ``cfg`` on :py:obj:`Config.SCHEMA`.</span>
|
||||||
|
<span class="sd"> Validation is done by :py:obj:`validate`."""</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">validate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg_schema</span><span class="p">,</span> <span class="n">cfg</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deprecated</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Config.update">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.update">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upd_cfg</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Update this configuration by ``upd_cfg``."""</span>
|
||||||
|
|
||||||
|
<span class="n">dict_deepupdate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="p">,</span> <span class="n">upd_cfg</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Config.default">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.default">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">default</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns default value of field ``name`` in ``self.cfg_schema``."""</span>
|
||||||
|
<span class="k">return</span> <span class="n">value</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg_schema</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Config.get">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.get">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">UNSET</span><span class="p">,</span> <span class="n">replace</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">)</span> <span class="o">-></span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns the value to which ``name`` points in the configuration.</span>
|
||||||
|
|
||||||
|
<span class="sd"> If there is no such ``name`` in the config and the ``default`` is</span>
|
||||||
|
<span class="sd"> :py:obj:`UNSET`, a :py:obj:`KeyError` is raised.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">parent</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_parent_dict</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">parent</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">UNSET</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">default</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">default</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">replace</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">val</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">val</span> <span class="o">%</span> <span class="bp">self</span>
|
||||||
|
<span class="k">return</span> <span class="n">val</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Config.set">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.set">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">val</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Set the value to which ``name`` points in the configuration.</span>
|
||||||
|
|
||||||
|
<span class="sd"> If there is no such ``name`` in the config, a :py:obj:`KeyError` is</span>
|
||||||
|
<span class="sd"> raised.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">parent</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_parent_dict</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="n">parent</span><span class="p">[</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span> <span class="o">=</span> <span class="n">val</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_get_parent_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
|
||||||
|
<span class="n">parent_name</span> <span class="o">=</span> <span class="s1">'.'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
|
||||||
|
<span class="k">if</span> <span class="n">parent_name</span><span class="p">:</span>
|
||||||
|
<span class="n">parent</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span><span class="p">(</span><span class="n">parent_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">parent</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span>
|
||||||
|
<span class="k">if</span> <span class="p">(</span><span class="n">parent</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">parent</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">parent_name</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">parent</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Config.path">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.path">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">path</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">UNSET</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get a :py:class:`pathlib.Path` object from a config string."""</span>
|
||||||
|
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">default</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">default</span>
|
||||||
|
<span class="k">return</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">val</span><span class="p">))</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Config.pyobj">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.config.Config.pyobj">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">pyobj</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">UNSET</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get python object referred by full qualiffied name (FQN) in the config</span>
|
||||||
|
<span class="sd"> string."""</span>
|
||||||
|
|
||||||
|
<span class="n">fqn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">fqn</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">default</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">default</span>
|
||||||
|
<span class="p">(</span><span class="n">modulename</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">fqn</span><span class="p">)</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">m</span> <span class="o">=</span> <span class="nb">__import__</span><span class="p">(</span><span class="n">modulename</span><span class="p">,</span> <span class="p">{},</span> <span class="p">{},</span> <span class="p">[</span><span class="n">name</span><span class="p">],</span> <span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">toml_load</span><span class="p">(</span><span class="n">file_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">):</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_name</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">tomllib</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">tomllib</span><span class="o">.</span><span class="n">TOMLDecodeError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">exc</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\t</span><span class="s1">'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="s1">' '</span><span class="p">)</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">file_name</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
|
||||||
|
<span class="k">raise</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># working with dictionaries</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">value</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">data_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns the value to which ``name`` points in the ``dat_dict``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> >>> data_dict = {</span>
|
||||||
|
<span class="sd"> "foo": {"bar": 1 },</span>
|
||||||
|
<span class="sd"> "bar": {"foo": 2 },</span>
|
||||||
|
<span class="sd"> "foobar": [1, 2, 3],</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
<span class="sd"> >>> value('foobar', data_dict)</span>
|
||||||
|
<span class="sd"> [1, 2, 3]</span>
|
||||||
|
<span class="sd"> >>> value('foo.bar', data_dict)</span>
|
||||||
|
<span class="sd"> 1</span>
|
||||||
|
<span class="sd"> >>> value('foo.bar.xxx', data_dict)</span>
|
||||||
|
<span class="sd"> <UNSET></span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">ret_val</span> <span class="o">=</span> <span class="n">data_dict</span>
|
||||||
|
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ret_val</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="n">ret_val</span> <span class="o">=</span> <span class="n">ret_val</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">part</span><span class="p">,</span> <span class="n">UNSET</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">ret_val</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
<span class="k">return</span> <span class="n">ret_val</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">validate</span><span class="p">(</span>
|
||||||
|
<span class="n">schema_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">data_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">SchemaIssue</span><span class="p">]]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Deep validation of dictionary in ``data_dict`` against dictionary in</span>
|
||||||
|
<span class="sd"> ``schema_dict``. Argument deprecated is a dictionary that maps deprecated</span>
|
||||||
|
<span class="sd"> configuration names to a messages::</span>
|
||||||
|
|
||||||
|
<span class="sd"> deprecated = {</span>
|
||||||
|
<span class="sd"> "foo.bar" : "config 'foo.bar' is deprecated, use 'bar.foo'",</span>
|
||||||
|
<span class="sd"> "..." : "..."</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
|
||||||
|
<span class="sd"> The function returns a python tuple ``(is_valid, issue_list)``:</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``is_valid``:</span>
|
||||||
|
<span class="sd"> A bool value indicating ``data_dict`` is valid or not.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``issue_list``:</span>
|
||||||
|
<span class="sd"> A list of messages (:py:obj:`SchemaIssue`) from the validation::</span>
|
||||||
|
|
||||||
|
<span class="sd"> [schema warn] data_dict: deprecated 'fontlib.foo': <DEPRECATED['foo.bar']></span>
|
||||||
|
<span class="sd"> [schema invalid] data_dict: key unknown 'fontlib.foo'</span>
|
||||||
|
<span class="sd"> [schema invalid] data_dict: type mismatch 'fontlib.foo': expected ..., is ...</span>
|
||||||
|
|
||||||
|
<span class="sd"> If ``schema_dict`` or ``data_dict`` is not a dictionary type a</span>
|
||||||
|
<span class="sd"> :py:obj:`SchemaIssue` is raised.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">is_valid</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">issue_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SchemaIssue</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">schema_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SchemaIssue</span><span class="p">(</span><span class="s1">'invalid'</span><span class="p">,</span> <span class="s2">"schema_dict is not a dict type"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SchemaIssue</span><span class="p">(</span><span class="s1">'invalid'</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"data_dict issue</span><span class="si">{</span><span class="s1">'.'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span><span class="si">}</span><span class="s2"> is not a dict type"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">is_valid</span><span class="p">,</span> <span class="n">issue_list</span> <span class="o">=</span> <span class="n">_validate</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">issue_list</span><span class="p">,</span> <span class="n">schema_dict</span><span class="p">,</span> <span class="n">data_dict</span><span class="p">,</span> <span class="n">deprecated</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">is_valid</span><span class="p">,</span> <span class="n">issue_list</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_validate</span><span class="p">(</span>
|
||||||
|
<span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
|
||||||
|
<span class="n">issue_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">SchemaIssue</span><span class="p">],</span>
|
||||||
|
<span class="n">schema_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span>
|
||||||
|
<span class="n">data_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span>
|
||||||
|
<span class="n">deprecated</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">SchemaIssue</span><span class="p">]]:</span>
|
||||||
|
|
||||||
|
<span class="n">is_valid</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">data_value</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span>
|
||||||
|
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">data_value</span> <span class="ow">in</span> <span class="n">data_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
|
||||||
|
<span class="n">names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="s1">'.'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">deprecated_msg</span> <span class="o">=</span> <span class="n">deprecated</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="c1"># print("XXX %s: key %s // data_value: %s" % (name, key, data_value))</span>
|
||||||
|
<span class="k">if</span> <span class="n">deprecated_msg</span><span class="p">:</span>
|
||||||
|
<span class="n">issue_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">SchemaIssue</span><span class="p">(</span><span class="s1">'warn'</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"data_dict '</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">': deprecated - </span><span class="si">{</span><span class="n">deprecated_msg</span><span class="si">}</span><span class="s2">"</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">schema_value</span> <span class="o">=</span> <span class="n">value</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">schema_dict</span><span class="p">)</span>
|
||||||
|
<span class="c1"># print("YYY %s: key %s // schema_value: %s" % (name, key, schema_value))</span>
|
||||||
|
<span class="k">if</span> <span class="n">schema_value</span> <span class="ow">is</span> <span class="n">UNSET</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">deprecated_msg</span><span class="p">:</span>
|
||||||
|
<span class="n">issue_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">SchemaIssue</span><span class="p">(</span><span class="s1">'invalid'</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"data_dict '</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">': key unknown in schema_dict"</span><span class="p">))</span>
|
||||||
|
<span class="n">is_valid</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">schema_value</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">type</span><span class="p">(</span><span class="n">data_value</span><span class="p">):</span> <span class="c1"># pylint: disable=unidiomatic-typecheck</span>
|
||||||
|
<span class="n">issue_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="n">SchemaIssue</span><span class="p">(</span>
|
||||||
|
<span class="s1">'invalid'</span><span class="p">,</span>
|
||||||
|
<span class="p">(</span><span class="sa">f</span><span class="s2">"data_dict: type mismatch '</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">':"</span> <span class="sa">f</span><span class="s2">" expected </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">schema_value</span><span class="p">)</span><span class="si">}</span><span class="s2">, is: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">data_value</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span><span class="p">),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">is_valid</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data_value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="n">_valid</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">_validate</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">issue_list</span><span class="p">,</span> <span class="n">schema_dict</span><span class="p">,</span> <span class="n">data_value</span><span class="p">,</span> <span class="n">deprecated</span><span class="p">)</span>
|
||||||
|
<span class="n">is_valid</span> <span class="o">=</span> <span class="n">is_valid</span> <span class="ow">and</span> <span class="n">_valid</span>
|
||||||
|
<span class="n">names</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">is_valid</span><span class="p">,</span> <span class="n">issue_list</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">dict_deepupdate</span><span class="p">(</span><span class="n">base_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">upd_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">],</span> <span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Deep-update of dictionary in ``base_dict`` by dictionary in ``upd_dict``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> For each ``upd_key`` & ``upd_val`` pair in ``upd_dict``:</span>
|
||||||
|
|
||||||
|
<span class="sd"> 0. If types of ``base_dict[upd_key]`` and ``upd_val`` do not match raise a</span>
|
||||||
|
<span class="sd"> :py:obj:`TypeError`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> 1. If ``base_dict[upd_key]`` is a dict: recursively deep-update it by ``upd_val``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> 2. If ``base_dict[upd_key]`` not exist: set ``base_dict[upd_key]`` from a</span>
|
||||||
|
<span class="sd"> (deep-) copy of ``upd_val``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> 3. If ``upd_val`` is a list, extend list in ``base_dict[upd_key]`` by the</span>
|
||||||
|
<span class="sd"> list in ``upd_val``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> 4. If ``upd_val`` is a set, update set in ``base_dict[upd_key]`` by set in</span>
|
||||||
|
<span class="sd"> ``upd_val``.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">base_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"argument 'base_dict' is not a dictionary type"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">upd_dict</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"argument 'upd_dict' is not a dictionary type"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">names</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">names</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">upd_key</span><span class="p">,</span> <span class="n">upd_val</span> <span class="ow">in</span> <span class="n">upd_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="c1"># For each upd_key & upd_val pair in upd_dict:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">upd_val</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">upd_key</span> <span class="ow">in</span> <span class="n">base_dict</span><span class="p">:</span>
|
||||||
|
<span class="c1"># if base_dict[upd_key] exists, recursively deep-update it</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">],</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"type mismatch </span><span class="si">{</span><span class="s1">'.'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span><span class="si">}</span><span class="s2">: is not a dict type in base_dict"</span><span class="p">)</span>
|
||||||
|
<span class="n">dict_deepupdate</span><span class="p">(</span>
|
||||||
|
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">],</span>
|
||||||
|
<span class="n">upd_val</span><span class="p">,</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
<span class="n">names</span>
|
||||||
|
<span class="o">+</span> <span class="p">[</span>
|
||||||
|
<span class="n">upd_key</span><span class="p">,</span>
|
||||||
|
<span class="p">],</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># if base_dict[upd_key] not exist, set base_dict[upd_key] from deepcopy of upd_val</span>
|
||||||
|
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">upd_val</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">upd_val</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">upd_key</span> <span class="ow">in</span> <span class="n">base_dict</span><span class="p">:</span>
|
||||||
|
<span class="c1"># if base_dict[upd_key] exists, base_dict[up_key] is extended by</span>
|
||||||
|
<span class="c1"># the list from upd_val</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"type mismatch </span><span class="si">{</span><span class="s1">'.'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span><span class="si">}</span><span class="s2">: is not a list type in base_dict"</span><span class="p">)</span>
|
||||||
|
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">upd_val</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># if base_dict[upd_key] doesn't exists, set base_dict[key] from a deepcopy of the</span>
|
||||||
|
<span class="c1"># list in upd_val.</span>
|
||||||
|
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">upd_val</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">upd_val</span><span class="p">,</span> <span class="nb">set</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">upd_key</span> <span class="ow">in</span> <span class="n">base_dict</span><span class="p">:</span>
|
||||||
|
<span class="c1"># if base_dict[upd_key] exists, base_dict[up_key] is updated by the set in upd_val</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">],</span> <span class="nb">set</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"type mismatch </span><span class="si">{</span><span class="s1">'.'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">names</span><span class="p">)</span><span class="si">}</span><span class="s2">: is not a set type in base_dict"</span><span class="p">)</span>
|
||||||
|
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">upd_val</span><span class="o">.</span><span class="n">copy</span><span class="p">())</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># if base_dict[upd_key] doesn't exists, set base_dict[upd_key] from a copy of the</span>
|
||||||
|
<span class="c1"># set in upd_val</span>
|
||||||
|
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">upd_val</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># for any other type of upd_val replace or add base_dict[upd_key] by a copy</span>
|
||||||
|
<span class="c1"># of upd_val</span>
|
||||||
|
<span class="n">base_dict</span><span class="p">[</span><span class="n">upd_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">upd_val</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
214
_modules/searx/botdetection/http_sec_fetch.html
Normal file
@@ -0,0 +1,214 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.botdetection.http_sec_fetch — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.botdetection.http_sec_fetch</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.botdetection.http_sec_fetch</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Method ``http_sec_fetch``</span>
|
||||||
|
<span class="sd">-------------------------</span>
|
||||||
|
|
||||||
|
<span class="sd">The ``http_sec_fetch`` method protect resources from web attacks with `Fetch</span>
|
||||||
|
<span class="sd">Metadata`_. A request is filtered out in case of:</span>
|
||||||
|
|
||||||
|
<span class="sd">- http header Sec-Fetch-Mode_ is invalid</span>
|
||||||
|
<span class="sd">- http header Sec-Fetch-Dest_ is invalid</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Fetch Metadata:</span>
|
||||||
|
<span class="sd"> https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Sec-Fetch-Dest:</span>
|
||||||
|
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/API/Request/destination</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Sec-Fetch-Mode:</span>
|
||||||
|
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/API/Request/mode</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">IPv4Network</span><span class="p">,</span>
|
||||||
|
<span class="n">IPv6Network</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">werkzeug</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="is_browser_supported">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.http_sec_fetch.is_browser_supported">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_browser_supported</span><span class="p">(</span><span class="n">user_agent</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Check if the browser supports Sec-Fetch headers.</span>
|
||||||
|
|
||||||
|
<span class="sd"> https://caniuse.com/mdn-http_headers_sec-fetch-dest</span>
|
||||||
|
<span class="sd"> https://caniuse.com/mdn-http_headers_sec-fetch-mode</span>
|
||||||
|
<span class="sd"> https://caniuse.com/mdn-http_headers_sec-fetch-site</span>
|
||||||
|
|
||||||
|
<span class="sd"> Supported browsers:</span>
|
||||||
|
<span class="sd"> - Chrome >= 80</span>
|
||||||
|
<span class="sd"> - Firefox >= 90</span>
|
||||||
|
<span class="sd"> - Safari >= 16.4</span>
|
||||||
|
<span class="sd"> - Edge (mirrors Chrome)</span>
|
||||||
|
<span class="sd"> - Opera (mirrors Chrome)</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">user_agent</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># Chrome/Chromium/Edge/Opera</span>
|
||||||
|
<span class="n">chrome_match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s1">'chrome/(\d+)'</span><span class="p">,</span> <span class="n">user_agent</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">chrome_match</span><span class="p">:</span>
|
||||||
|
<span class="n">version</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">chrome_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">version</span> <span class="o">>=</span> <span class="mi">80</span>
|
||||||
|
|
||||||
|
<span class="c1"># Firefox</span>
|
||||||
|
<span class="n">firefox_match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s1">'firefox/(\d+)'</span><span class="p">,</span> <span class="n">user_agent</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">firefox_match</span><span class="p">:</span>
|
||||||
|
<span class="n">version</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">firefox_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">version</span> <span class="o">>=</span> <span class="mi">90</span>
|
||||||
|
|
||||||
|
<span class="c1"># Safari</span>
|
||||||
|
<span class="n">safari_match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s1">'version/(\d+)\.(\d+)'</span><span class="p">,</span> <span class="n">user_agent</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">safari_match</span><span class="p">:</span>
|
||||||
|
<span class="n">major</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">safari_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
|
||||||
|
<span class="n">minor</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">safari_match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">major</span> <span class="o">></span> <span class="mi">16</span> <span class="ow">or</span> <span class="p">(</span><span class="n">major</span> <span class="o">==</span> <span class="mi">16</span> <span class="ow">and</span> <span class="n">minor</span> <span class="o">>=</span> <span class="mi">4</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">False</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">filter_request</span><span class="p">(</span>
|
||||||
|
<span class="n">network</span><span class="p">:</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">,</span>
|
||||||
|
<span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">,</span>
|
||||||
|
<span class="n">cfg</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="n">werkzeug</span><span class="o">.</span><span class="n">Response</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">request</span><span class="o">.</span><span class="n">is_secure</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||||
|
<span class="s2">"Sec-Fetch cannot be verified for non-secure requests (HTTP headers are not set/sent by the client)."</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># Only check Sec-Fetch headers for supported browsers</span>
|
||||||
|
<span class="n">user_agent</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'User-Agent'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">is_browser_supported</span><span class="p">(</span><span class="n">user_agent</span><span class="p">):</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"Sec-Fetch-Mode"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'navigate'</span><span class="p">,</span> <span class="s1">'cors'</span><span class="p">):</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"invalid Sec-Fetch-Mode '</span><span class="si">%s</span><span class="s2">'"</span><span class="p">,</span> <span class="n">val</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">redirect</span><span class="p">(</span><span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">'index'</span><span class="p">),</span> <span class="n">code</span><span class="o">=</span><span class="mi">302</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"Sec-Fetch-Site"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'same-origin'</span><span class="p">,</span> <span class="s1">'same-site'</span><span class="p">,</span> <span class="s1">'none'</span><span class="p">):</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"invalid Sec-Fetch-Site '</span><span class="si">%s</span><span class="s2">'"</span><span class="p">,</span> <span class="n">val</span><span class="p">)</span>
|
||||||
|
<span class="n">flask</span><span class="o">.</span><span class="n">redirect</span><span class="p">(</span><span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">'index'</span><span class="p">),</span> <span class="n">code</span><span class="o">=</span><span class="mi">302</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"Sec-Fetch-Dest"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'document'</span><span class="p">,</span> <span class="s1">'empty'</span><span class="p">):</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"invalid Sec-Fetch-Dest '</span><span class="si">%s</span><span class="s2">'"</span><span class="p">,</span> <span class="n">val</span><span class="p">)</span>
|
||||||
|
<span class="n">flask</span><span class="o">.</span><span class="n">redirect</span><span class="p">(</span><span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">'index'</span><span class="p">),</span> <span class="n">code</span><span class="o">=</span><span class="mi">302</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
194
_modules/searx/botdetection/ip_lists.html
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.botdetection.ip_lists — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.botdetection.ip_lists</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.botdetection.ip_lists</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">""".. _botdetection.ip_lists:</span>
|
||||||
|
|
||||||
|
<span class="sd">Method ``ip_lists``</span>
|
||||||
|
<span class="sd">-------------------</span>
|
||||||
|
|
||||||
|
<span class="sd">The ``ip_lists`` method implements :py:obj:`block-list <block_ip>` and</span>
|
||||||
|
<span class="sd">:py:obj:`pass-list <pass_ip>`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: toml</span>
|
||||||
|
|
||||||
|
<span class="sd"> [botdetection.ip_lists]</span>
|
||||||
|
|
||||||
|
<span class="sd"> pass_ip = [</span>
|
||||||
|
<span class="sd"> '167.235.158.251', # IPv4 of check.searx.space</span>
|
||||||
|
<span class="sd"> '192.168.0.0/16', # IPv4 private network</span>
|
||||||
|
<span class="sd"> 'fe80::/10', # IPv6 linklocal</span>
|
||||||
|
<span class="sd"> ]</span>
|
||||||
|
|
||||||
|
<span class="sd"> block_ip = [</span>
|
||||||
|
<span class="sd"> '93.184.216.34', # IPv4 of example.org</span>
|
||||||
|
<span class="sd"> '257.1.1.1', # invalid IP --> will be ignored, logged in ERROR class</span>
|
||||||
|
<span class="sd"> ]</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Tuple</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">ip_network</span><span class="p">,</span>
|
||||||
|
<span class="n">IPv4Address</span><span class="p">,</span>
|
||||||
|
<span class="n">IPv6Address</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'ip_limit'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">SEARXNG_ORG</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="c1"># https://github.com/searxng/searxng/pull/2484#issuecomment-1576639195</span>
|
||||||
|
<span class="s1">'167.235.158.251'</span><span class="p">,</span> <span class="c1"># IPv4 check.searx.space</span>
|
||||||
|
<span class="s1">'2a01:04f8:1c1c:8fc2::/64'</span><span class="p">,</span> <span class="c1"># IPv6 check.searx.space</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="sd">"""Passlist of IPs from the SearXNG organization, e.g. `check.searx.space`."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="pass_ip">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.ip_lists.pass_ip">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">pass_ip</span><span class="p">(</span><span class="n">real_ip</span><span class="p">:</span> <span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">)</span> <span class="o">-></span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Checks if the IP on the subnet is in one of the members of the</span>
|
||||||
|
<span class="sd"> ``botdetection.ip_lists.pass_ip`` list.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'botdetection.ip_lists.pass_searxng_org'</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="k">for</span> <span class="n">net</span> <span class="ow">in</span> <span class="n">SEARXNG_ORG</span><span class="p">:</span>
|
||||||
|
<span class="n">net</span> <span class="o">=</span> <span class="n">ip_network</span><span class="p">(</span><span class="n">net</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">real_ip</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="n">net</span><span class="o">.</span><span class="n">version</span> <span class="ow">and</span> <span class="n">real_ip</span> <span class="ow">in</span> <span class="n">net</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"IP matches </span><span class="si">{</span><span class="n">net</span><span class="o">.</span><span class="n">compressed</span><span class="si">}</span><span class="s2"> in SEARXNG_ORG list."</span>
|
||||||
|
<span class="k">return</span> <span class="n">ip_is_subnet_of_member_in_list</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="s1">'botdetection.ip_lists.pass_ip'</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="block_ip">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.ip_lists.block_ip">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">block_ip</span><span class="p">(</span><span class="n">real_ip</span><span class="p">:</span> <span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">)</span> <span class="o">-></span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Checks if the IP on the subnet is in one of the members of the</span>
|
||||||
|
<span class="sd"> ``botdetection.ip_lists.block_ip`` list.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">block</span><span class="p">,</span> <span class="n">msg</span> <span class="o">=</span> <span class="n">ip_is_subnet_of_member_in_list</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="s1">'botdetection.ip_lists.block_ip'</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">block</span><span class="p">:</span>
|
||||||
|
<span class="n">msg</span> <span class="o">+=</span> <span class="s2">" To remove IP from list, please contact the maintainer of the service."</span>
|
||||||
|
<span class="k">return</span> <span class="n">block</span><span class="p">,</span> <span class="n">msg</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">ip_is_subnet_of_member_in_list</span><span class="p">(</span>
|
||||||
|
<span class="n">real_ip</span><span class="p">:</span> <span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">list_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="k">for</span> <span class="n">net</span> <span class="ow">in</span> <span class="n">cfg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">list_name</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="p">[]):</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">net</span> <span class="o">=</span> <span class="n">ip_network</span><span class="p">(</span><span class="n">net</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"invalid IP </span><span class="si">%s</span><span class="s2"> in </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">net</span><span class="p">,</span> <span class="n">list_name</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="n">real_ip</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="n">net</span><span class="o">.</span><span class="n">version</span> <span class="ow">and</span> <span class="n">real_ip</span> <span class="ow">in</span> <span class="n">net</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"IP matches </span><span class="si">{</span><span class="n">net</span><span class="o">.</span><span class="n">compressed</span><span class="si">}</span><span class="s2"> in </span><span class="si">{</span><span class="n">list_name</span><span class="si">}</span><span class="s2">."</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"IP is not a member of an item in the f</span><span class="si">{</span><span class="n">list_name</span><span class="si">}</span><span class="s2"> list"</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
272
_modules/searx/botdetection/link_token.html
Normal file
@@ -0,0 +1,272 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.botdetection.link_token — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.botdetection.link_token</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.botdetection.link_token</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Method ``link_token``</span>
|
||||||
|
<span class="sd">---------------------</span>
|
||||||
|
|
||||||
|
<span class="sd">The ``link_token`` method evaluates a request as :py:obj:`suspicious</span>
|
||||||
|
<span class="sd"><is_suspicious>` if the URL ``/client<token>.css`` is not requested by the</span>
|
||||||
|
<span class="sd">client. By adding a random component (the token) in the URL, a bot can not send</span>
|
||||||
|
<span class="sd">a ping by request a static URL.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> This method requires a valkey DB and needs a HTTP X-Forwarded-For_ header.</span>
|
||||||
|
|
||||||
|
<span class="sd">To get in use of this method a flask URL route needs to be added:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> @app.route('/client<token>.css', methods=['GET', 'POST'])</span>
|
||||||
|
<span class="sd"> def client_token(token=None):</span>
|
||||||
|
<span class="sd"> link_token.ping(request, token)</span>
|
||||||
|
<span class="sd"> return Response('', mimetype='text/css')</span>
|
||||||
|
|
||||||
|
<span class="sd">And in the HTML template from flask a stylesheet link is needed (the value of</span>
|
||||||
|
<span class="sd">``link_token`` comes from :py:obj:`get_token`):</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: html</span>
|
||||||
|
|
||||||
|
<span class="sd"> <link rel="stylesheet"</span>
|
||||||
|
<span class="sd"> href="{{ url_for('client_token', token=link_token) }}"</span>
|
||||||
|
<span class="sd"> type="text/css" ></span>
|
||||||
|
|
||||||
|
<span class="sd">.. _X-Forwarded-For:</span>
|
||||||
|
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">IPv4Network</span><span class="p">,</span>
|
||||||
|
<span class="n">IPv6Network</span><span class="p">,</span>
|
||||||
|
<span class="n">ip_address</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.valkeylib</span><span class="w"> </span><span class="kn">import</span> <span class="n">secret_hash</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._helpers</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">get_network</span><span class="p">,</span>
|
||||||
|
<span class="n">logger</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">valkeydb</span>
|
||||||
|
|
||||||
|
<span class="n">TOKEN_LIVE_TIME</span> <span class="o">=</span> <span class="mi">600</span>
|
||||||
|
<span class="sd">"""Lifetime (sec) of limiter's CSS token."""</span>
|
||||||
|
|
||||||
|
<span class="n">PING_LIVE_TIME</span> <span class="o">=</span> <span class="mi">3600</span>
|
||||||
|
<span class="sd">"""Lifetime (sec) of the ping-key from a client (request)"""</span>
|
||||||
|
|
||||||
|
<span class="n">PING_KEY</span> <span class="o">=</span> <span class="s1">'SearXNG_limiter.ping'</span>
|
||||||
|
<span class="sd">"""Prefix of all ping-keys generated by :py:obj:`get_ping_key`"""</span>
|
||||||
|
|
||||||
|
<span class="n">TOKEN_KEY</span> <span class="o">=</span> <span class="s1">'SearXNG_limiter.token'</span>
|
||||||
|
<span class="sd">"""Key for which the current token is stored in the DB"""</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'botdetection.link_token'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="is_suspicious">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.link_token.is_suspicious">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_suspicious</span><span class="p">(</span><span class="n">network</span><span class="p">:</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">,</span> <span class="n">renew</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Checks whether a valid ping is exists for this (client) network, if not</span>
|
||||||
|
<span class="sd"> this request is rated as *suspicious*. If a valid ping exists and argument</span>
|
||||||
|
<span class="sd"> ``renew`` is ``True`` the expire time of this ping is reset to</span>
|
||||||
|
<span class="sd"> :py:obj:`PING_LIVE_TIME`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">valkey_client</span> <span class="o">=</span> <span class="n">valkeydb</span><span class="o">.</span><span class="n">get_valkey_client</span><span class="p">()</span>
|
||||||
|
<span class="n">ping_key</span> <span class="o">=</span> <span class="n">get_ping_key</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">request</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">valkey_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">ping_key</span><span class="p">):</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"missing ping (IP: </span><span class="si">%s</span><span class="s2">) / request: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">ping_key</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">renew</span><span class="p">:</span>
|
||||||
|
<span class="n">valkey_client</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">ping_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">ex</span><span class="o">=</span><span class="n">PING_LIVE_TIME</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"found ping for (client) network </span><span class="si">%s</span><span class="s2"> -> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">ping_key</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ping">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.link_token.ping">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">ping</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">,</span> <span class="n">token</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""This function is called by a request to URL ``/client<token>.css``. If</span>
|
||||||
|
<span class="sd"> ``token`` is valid a :py:obj:`PING_KEY` for the client is stored in the DB.</span>
|
||||||
|
<span class="sd"> The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">valkey_client</span> <span class="o">=</span> <span class="n">valkeydb</span><span class="o">.</span><span class="n">get_valkey_client</span><span class="p">()</span>
|
||||||
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get_global_cfg</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">token_is_valid</span><span class="p">(</span><span class="n">token</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="n">real_ip</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">network</span> <span class="o">=</span> <span class="n">get_network</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">ping_key</span> <span class="o">=</span> <span class="n">get_ping_key</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">request</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||||
|
<span class="s2">"store ping_key for (client) network </span><span class="si">%s</span><span class="s2"> (IP </span><span class="si">%s</span><span class="s2">) -> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">real_ip</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">ping_key</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">valkey_client</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">ping_key</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">ex</span><span class="o">=</span><span class="n">PING_LIVE_TIME</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_ping_key">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.link_token.get_ping_key">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_ping_key</span><span class="p">(</span><span class="n">network</span><span class="p">:</span> <span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Generates a hashed key that fits (more or less) to a *WEB-browser</span>
|
||||||
|
<span class="sd"> session* in a network."""</span>
|
||||||
|
<span class="k">return</span> <span class="p">(</span>
|
||||||
|
<span class="n">PING_KEY</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"["</span>
|
||||||
|
<span class="o">+</span> <span class="n">secret_hash</span><span class="p">(</span>
|
||||||
|
<span class="n">network</span><span class="o">.</span><span class="n">compressed</span> <span class="o">+</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Accept-Language'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> <span class="o">+</span> <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'User-Agent'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"]"</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">token_is_valid</span><span class="p">(</span><span class="n">token</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="n">valid</span> <span class="o">=</span> <span class="n">token</span> <span class="o">==</span> <span class="n">get_token</span><span class="p">()</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"token is valid --> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">valid</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">valid</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_token">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.link_token.get_token">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_token</span><span class="p">()</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns current token. If there is no currently active token a new token</span>
|
||||||
|
<span class="sd"> is generated randomly and stored in the Valkey DB. Without without a</span>
|
||||||
|
<span class="sd"> database connection, string "12345678" is returned.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - :py:obj:`TOKEN_LIVE_TIME`</span>
|
||||||
|
<span class="sd"> - :py:obj:`TOKEN_KEY`</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">valkey_client</span> <span class="o">=</span> <span class="n">valkeydb</span><span class="o">.</span><span class="n">get_valkey_client</span><span class="p">()</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||||
|
<span class="c1"># This function is also called when limiter is inactive / no valkey DB</span>
|
||||||
|
<span class="c1"># (see render function in webapp.py)</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'12345678'</span>
|
||||||
|
|
||||||
|
<span class="n">token</span> <span class="o">=</span> <span class="n">valkey_client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">TOKEN_KEY</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">token</span><span class="p">:</span>
|
||||||
|
<span class="n">token</span> <span class="o">=</span> <span class="n">token</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'UTF-8'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">token</span> <span class="o">=</span> <span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">string</span><span class="o">.</span><span class="n">ascii_lowercase</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">16</span><span class="p">))</span>
|
||||||
|
<span class="n">valkey_client</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">TOKEN_KEY</span><span class="p">,</span> <span class="n">token</span><span class="p">,</span> <span class="n">ex</span><span class="o">=</span><span class="n">TOKEN_LIVE_TIME</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">token</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
283
_modules/searx/botdetection/trusted_proxies.html
Normal file
@@ -0,0 +1,283 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.botdetection.trusted_proxies — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.botdetection.trusted_proxies</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.botdetection.trusted_proxies</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Implementation of a middleware to determine the real IP of an HTTP request</span>
|
||||||
|
<span class="sd">(:py:obj:`flask.request.remote_addr`) behind a proxy chain."""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">abc</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="n">IPv4Address</span><span class="p">,</span> <span class="n">IPv6Address</span><span class="p">,</span> <span class="n">ip_address</span><span class="p">,</span> <span class="n">ip_network</span><span class="p">,</span> <span class="n">IPv4Network</span><span class="p">,</span> <span class="n">IPv6Network</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">werkzeug.http</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse_list_header</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">log_error_only_once</span><span class="p">,</span> <span class="n">logger</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">_typeshed.wsgi</span><span class="w"> </span><span class="kn">import</span> <span class="n">StartResponse</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">_typeshed.wsgi</span><span class="w"> </span><span class="kn">import</span> <span class="n">WSGIApplication</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">_typeshed.wsgi</span><span class="w"> </span><span class="kn">import</span> <span class="n">WSGIEnvironment</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ProxyFix">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.botdetection.html#searx.botdetection.ProxyFix">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ProxyFix</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A middleware like the ProxyFix_ class, where the ``x_for`` argument is</span>
|
||||||
|
<span class="sd"> replaced by a method that determines the number of trusted proxies via the</span>
|
||||||
|
<span class="sd"> ``botdetection.trusted_proxies`` setting.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. sidebar:: :py:obj:`flask.Request.remote_addr`</span>
|
||||||
|
|
||||||
|
<span class="sd"> SearXNG uses Werkzeug's ProxyFix_ (with it default ``x_for=1``).</span>
|
||||||
|
|
||||||
|
<span class="sd"> The remote IP (:py:obj:`flask.Request.remote_addr`) of the request is taken</span>
|
||||||
|
<span class="sd"> from (first match):</span>
|
||||||
|
|
||||||
|
<span class="sd"> - X-Forwarded-For_: If the header is set, the first untrusted IP that comes</span>
|
||||||
|
<span class="sd"> before the IPs that are still part of the ``botdetection.trusted_proxies``</span>
|
||||||
|
<span class="sd"> is used.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - `X-Real-IP <https://github.com/searxng/searxng/issues/1237#issuecomment-1147564516>`__:</span>
|
||||||
|
<span class="sd"> If X-Forwarded-For_ is not set, `X-Real-IP` is used</span>
|
||||||
|
<span class="sd"> (``botdetection.trusted_proxies`` is ignored).</span>
|
||||||
|
|
||||||
|
<span class="sd"> If none of the header is set, the REMOTE_ADDR_ from the WSGI layer is used.</span>
|
||||||
|
<span class="sd"> If (for whatever reasons) none IP can be determined, an error message is</span>
|
||||||
|
<span class="sd"> displayed and ``100::`` is used instead (:rfc:`6666`).</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _ProxyFix:</span>
|
||||||
|
<span class="sd"> https://werkzeug.palletsprojects.com/middleware/proxy_fix/</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _X-Forwarded-For:</span>
|
||||||
|
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _REMOTE_ADDR:</span>
|
||||||
|
<span class="sd"> https://wsgi.readthedocs.io/en/latest/proposals-2.0.html#making-some-keys-required</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wsgi_app</span><span class="p">:</span> <span class="s2">"WSGIApplication"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">wsgi_app</span> <span class="o">=</span> <span class="n">wsgi_app</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">trusted_proxies</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">]:</span>
|
||||||
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">get_global_cfg</span><span class="p">()</span>
|
||||||
|
<span class="n">proxy_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"botdetection.trusted_proxies"</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="p">[])</span>
|
||||||
|
<span class="k">return</span> <span class="p">[</span><span class="n">ip_network</span><span class="p">(</span><span class="n">net</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="k">for</span> <span class="n">net</span> <span class="ow">in</span> <span class="n">proxy_list</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">trusted_remote_addr</span><span class="p">(</span>
|
||||||
|
<span class="bp">self</span><span class="p">,</span>
|
||||||
|
<span class="n">x_forwarded_for</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">],</span>
|
||||||
|
<span class="n">trusted_proxies</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">IPv4Network</span> <span class="o">|</span> <span class="n">IPv6Network</span><span class="p">],</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="c1"># always rtl</span>
|
||||||
|
<span class="k">for</span> <span class="n">addr</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="n">x_forwarded_for</span><span class="p">):</span>
|
||||||
|
<span class="n">trust</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">net</span> <span class="ow">in</span> <span class="n">trusted_proxies</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">addr</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="n">net</span><span class="o">.</span><span class="n">version</span> <span class="ow">and</span> <span class="n">addr</span> <span class="ow">in</span> <span class="n">net</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"trust proxy </span><span class="si">%s</span><span class="s2"> (member of </span><span class="si">%s</span><span class="s2">)"</span><span class="p">,</span> <span class="n">addr</span><span class="p">,</span> <span class="n">net</span><span class="p">)</span>
|
||||||
|
<span class="n">trust</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
|
||||||
|
<span class="c1"># client address</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">trust</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">addr</span><span class="o">.</span><span class="n">compressed</span>
|
||||||
|
|
||||||
|
<span class="c1"># fallback to first address</span>
|
||||||
|
<span class="k">return</span> <span class="n">x_forwarded_for</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">compressed</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">environ</span><span class="p">:</span> <span class="s2">"WSGIEnvironment"</span><span class="p">,</span> <span class="n">start_response</span><span class="p">:</span> <span class="s2">"StartResponse"</span><span class="p">)</span> <span class="o">-></span> <span class="n">abc</span><span class="o">.</span><span class="n">Iterable</span><span class="p">[</span><span class="nb">bytes</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-statements</span>
|
||||||
|
|
||||||
|
<span class="n">trusted_proxies</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">trusted_proxies</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># We do not rely on the REMOTE_ADDR from the WSGI environment / the</span>
|
||||||
|
<span class="c1"># variable is first removed from the WSGI environment and explicitly set</span>
|
||||||
|
<span class="c1"># in this function!</span>
|
||||||
|
|
||||||
|
<span class="n">orig_remote_addr</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">environ</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"REMOTE_ADDR"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Validate the IPs involved in this game and delete all invalid ones</span>
|
||||||
|
<span class="c1"># from the WSGI environment.</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">orig_remote_addr</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">addr</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">orig_remote_addr</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">addr</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="mi">6</span> <span class="ow">and</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span><span class="p">:</span>
|
||||||
|
<span class="n">addr</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span>
|
||||||
|
<span class="n">orig_remote_addr</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">compressed</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"REMOTE_ADDR: </span><span class="si">%s</span><span class="s2"> / discard REMOTE_ADDR from WSGI environment"</span><span class="p">,</span> <span class="n">exc</span><span class="p">)</span>
|
||||||
|
<span class="n">orig_remote_addr</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">x_real_ip</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"HTTP_X_REAL_IP"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">x_real_ip</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">addr</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">x_real_ip</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">addr</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="mi">6</span> <span class="ow">and</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span><span class="p">:</span>
|
||||||
|
<span class="n">addr</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span>
|
||||||
|
<span class="n">x_real_ip</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">compressed</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"X-Real-IP: </span><span class="si">%s</span><span class="s2"> / discard HTTP_X_REAL_IP from WSGI environment"</span><span class="p">,</span> <span class="n">exc</span><span class="p">)</span>
|
||||||
|
<span class="n">environ</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"HTTP_X_REAL_IP"</span><span class="p">)</span>
|
||||||
|
<span class="n">x_real_ip</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">x_forwarded_for</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">IPv4Address</span> <span class="o">|</span> <span class="n">IPv6Address</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">if</span> <span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"HTTP_X_FORWARDED_FOR"</span><span class="p">):</span>
|
||||||
|
<span class="k">for</span> <span class="n">x_for_ip</span> <span class="ow">in</span> <span class="n">parse_list_header</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"HTTP_X_FORWARDED_FOR"</span><span class="p">))):</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">addr</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">x_for_ip</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"X-Forwarded-For: </span><span class="si">%s</span><span class="s2"> / discard HTTP_X_FORWARDED_FOR from WSGI environment"</span><span class="p">,</span> <span class="n">exc</span><span class="p">)</span>
|
||||||
|
<span class="n">environ</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"HTTP_X_FORWARDED_FOR"</span><span class="p">)</span>
|
||||||
|
<span class="n">x_forwarded_for</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">addr</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="mi">6</span> <span class="ow">and</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span><span class="p">:</span>
|
||||||
|
<span class="n">addr</span> <span class="o">=</span> <span class="n">addr</span><span class="o">.</span><span class="n">ipv4_mapped</span>
|
||||||
|
<span class="n">x_forwarded_for</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">addr</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># log questionable WSGI environments</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">x_forwarded_for</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">x_real_ip</span><span class="p">:</span>
|
||||||
|
<span class="n">log_error_only_once</span><span class="p">(</span><span class="s2">"X-Forwarded-For nor X-Real-IP header is set!"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">x_forwarded_for</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">trusted_proxies</span><span class="p">:</span>
|
||||||
|
<span class="n">log_error_only_once</span><span class="p">(</span><span class="s2">"missing botdetection.trusted_proxies config"</span><span class="p">)</span>
|
||||||
|
<span class="c1"># without trusted_proxies, this variable is useless for determining</span>
|
||||||
|
<span class="c1"># the real IP</span>
|
||||||
|
<span class="n">x_forwarded_for</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="c1"># securing the WSGI environment variables that are adjusted</span>
|
||||||
|
|
||||||
|
<span class="n">environ</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">"botdetection.trusted_proxies.orig"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"REMOTE_ADDR"</span><span class="p">:</span> <span class="n">orig_remote_addr</span><span class="p">}})</span>
|
||||||
|
|
||||||
|
<span class="c1"># determine *the real IP*</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">x_forwarded_for</span><span class="p">:</span>
|
||||||
|
<span class="n">environ</span><span class="p">[</span><span class="s2">"REMOTE_ADDR"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">trusted_remote_addr</span><span class="p">(</span><span class="n">x_forwarded_for</span><span class="p">,</span> <span class="n">trusted_proxies</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">x_real_ip</span><span class="p">:</span>
|
||||||
|
<span class="n">environ</span><span class="p">[</span><span class="s2">"REMOTE_ADDR"</span><span class="p">]</span> <span class="o">=</span> <span class="n">x_real_ip</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">orig_remote_addr</span><span class="p">:</span>
|
||||||
|
<span class="n">environ</span><span class="p">[</span><span class="s2">"REMOTE_ADDR"</span><span class="p">]</span> <span class="o">=</span> <span class="n">orig_remote_addr</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"No remote IP could be determined, use black-hole address: 100::"</span><span class="p">)</span>
|
||||||
|
<span class="n">environ</span><span class="p">[</span><span class="s2">"REMOTE_ADDR"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"100::"</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">_</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">environ</span><span class="p">[</span><span class="s2">"REMOTE_ADDR"</span><span class="p">])</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"REMOTE_ADDR: </span><span class="si">%s</span><span class="s2">, use black-hole address: 100::"</span><span class="p">,</span> <span class="n">exc</span><span class="p">)</span>
|
||||||
|
<span class="n">environ</span><span class="p">[</span><span class="s2">"REMOTE_ADDR"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"100::"</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"final REMOTE_ADDR is: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">environ</span><span class="p">[</span><span class="s2">"REMOTE_ADDR"</span><span class="p">])</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">wsgi_app</span><span class="p">(</span><span class="n">environ</span><span class="p">,</span> <span class="n">start_response</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
175
_modules/searx/brand.html
Normal file
@@ -0,0 +1,175 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.brand — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.brand</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.brand</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Implementations needed for a branding of SearXNG."""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
|
||||||
|
<span class="c1"># Struct fields aren't discovered in Python 3.14</span>
|
||||||
|
<span class="c1"># - https://github.com/searxng/searxng/issues/5284</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"SettingsBrand"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">BrandCustom</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">forbid_unknown_fields</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Custom settings in the brand section."""</span>
|
||||||
|
|
||||||
|
<span class="n">links</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Custom entries in the footer of the WEB page: ``[title]: [link]``"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SettingsBrand">
|
||||||
|
<a class="viewcode-back" href="../../admin/settings/settings_brand.html#searx.brand.SettingsBrand">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SettingsBrand</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">forbid_unknown_fields</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Options for configuring brand properties.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> brand:</span>
|
||||||
|
<span class="sd"> issue_url: https://github.com/searxng/searxng/issues</span>
|
||||||
|
<span class="sd"> docs_url: https://docs.searxng.org</span>
|
||||||
|
<span class="sd"> public_instances: https://searx.space</span>
|
||||||
|
<span class="sd"> wiki_url: https://github.com/searxng/searxng/wiki</span>
|
||||||
|
|
||||||
|
<span class="sd"> custom:</span>
|
||||||
|
<span class="sd"> links:</span>
|
||||||
|
<span class="sd"> Uptime: https://uptime.searxng.org/history/example-org</span>
|
||||||
|
<span class="sd"> About: https://example.org/user/about.html</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">issue_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"https://github.com/searxng/searxng/issues"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""If you host your own issue tracker change this URL."""</span>
|
||||||
|
|
||||||
|
<span class="n">docs_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"https://docs.searxng.org"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""If you host your own documentation change this URL."""</span>
|
||||||
|
|
||||||
|
<span class="n">public_instances</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"https://searx.space"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""If you host your own https://searx.space change this URL."""</span>
|
||||||
|
|
||||||
|
<span class="n">wiki_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"https://github.com/searxng/searxng/wiki"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Link to your wiki (or ``false``)"""</span>
|
||||||
|
|
||||||
|
<span class="n">custom</span><span class="p">:</span> <span class="n">BrandCustom</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">BrandCustom</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Optional customizing.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. autoclass:: searx.brand.BrandCustom</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="c1"># new_issue_url is a hackish solution tailored for only one hoster (GH). As</span>
|
||||||
|
<span class="c1"># long as we don't have a more general solution, we should support it in the</span>
|
||||||
|
<span class="c1"># given function, but it should not be expanded further.</span>
|
||||||
|
|
||||||
|
<span class="n">new_issue_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"https://github.com/searxng/searxng/issues/new"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""If you host your own issue tracker not on GitHub, then unset this URL.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Note: This URL will create a pre-filled GitHub bug report form for an</span>
|
||||||
|
<span class="sd"> engine. Since this feature is implemented only for GH (and limited to</span>
|
||||||
|
<span class="sd"> engines), it will probably be replaced by another solution in the near</span>
|
||||||
|
<span class="sd"> future.</span>
|
||||||
|
<span class="sd"> """</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
642
_modules/searx/cache.html
Normal file
@@ -0,0 +1,642 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.cache — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.cache</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.cache</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="sd">"""Implementation of caching solutions.</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`searx.cache.ExpireCache` and its :py:obj:`searx.cache.ExpireCacheCfg`</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"ExpireCacheCfg"</span><span class="p">,</span> <span class="s2">"ExpireCacheStats"</span><span class="p">,</span> <span class="s2">"ExpireCache"</span><span class="p">,</span> <span class="s2">"ExpireCacheSQLite"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Iterator</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">dataclasses</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">datetime</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">hmac</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">pickle</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">sqlite3</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">sqlitedb</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_setting</span>
|
||||||
|
|
||||||
|
<span class="n">log</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s2">"cache"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">CacheRowType</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">TypeAlias</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheCfg">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheCfg">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ExpireCacheCfg</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Configuration of a :py:obj:`ExpireCache` cache."""</span>
|
||||||
|
|
||||||
|
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the cache."""</span>
|
||||||
|
|
||||||
|
<span class="n">db_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL of the SQLite DB, the path to the database file. If unset a default</span>
|
||||||
|
<span class="sd"> DB will be created in `/tmp/sxng_cache_{self.name}.db`"""</span>
|
||||||
|
|
||||||
|
<span class="n">MAX_VALUE_LEN</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">10</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Max length of a *serialized* value."""</span>
|
||||||
|
|
||||||
|
<span class="n">MAXHOLD_TIME</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span> <span class="c1"># 7 days</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Hold time (default in sec.), after which a value is removed from the cache."""</span>
|
||||||
|
|
||||||
|
<span class="n">MAINTENANCE_PERIOD</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="c1"># 2h</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to</span>
|
||||||
|
<span class="sd"> ``auto``."""</span>
|
||||||
|
|
||||||
|
<span class="n">MAINTENANCE_MODE</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"auto"</span><span class="p">,</span> <span class="s2">"off"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"auto"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Type of maintenance mode</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``auto``:</span>
|
||||||
|
<span class="sd"> Maintenance is carried out automatically as part of the maintenance</span>
|
||||||
|
<span class="sd"> intervals (:py:obj:`MAINTENANCE_PERIOD`); no external process is required.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``off``:</span>
|
||||||
|
<span class="sd"> Maintenance is switched off and must be carried out by an external process</span>
|
||||||
|
<span class="sd"> if required.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">password</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"server.secret_key"</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">()</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Password used by :py:obj:`ExpireCache.secret_hash`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The default password is taken from :ref:`secret_key <server.secret_key>`.</span>
|
||||||
|
<span class="sd"> When the password is changed, the hashed keys in the cache can no longer be</span>
|
||||||
|
<span class="sd"> used, which is why all values in the cache are deleted when the password is</span>
|
||||||
|
<span class="sd"> changed.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="c1"># if db_url is unset, use a default DB in /tmp/sxng_cache_{name}.db</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">db_url</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">db_url</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">gettempdir</span><span class="p">()</span> <span class="o">+</span> <span class="n">os</span><span class="o">.</span><span class="n">sep</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">"sxng_cache_</span><span class="si">{</span><span class="n">ExpireCache</span><span class="o">.</span><span class="n">normalize_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span><span class="si">}</span><span class="s2">.db"</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheStats">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheStats">[docs]</a>
|
||||||
|
<span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ExpireCacheStats</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Dataclass which provides information on the status of the cache."""</span>
|
||||||
|
|
||||||
|
<span class="n">cached_items</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">CacheRowType</span><span class="p">]]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Values in the cache mapped by context name.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> {</span>
|
||||||
|
<span class="sd"> "context name": [</span>
|
||||||
|
<span class="sd"> ("foo key": "foo value", <expire>),</span>
|
||||||
|
<span class="sd"> ("bar key": "bar value", <expire>),</span>
|
||||||
|
<span class="sd"> # ...</span>
|
||||||
|
<span class="sd"> ],</span>
|
||||||
|
<span class="sd"> # ...</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">report</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="n">c_ctx</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="n">c_kv</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="n">lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">ctx_name</span><span class="p">,</span> <span class="n">kv_list</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cached_items</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">c_ctx</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">kv_list</span><span class="p">:</span>
|
||||||
|
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"[</span><span class="si">{</span><span class="n">ctx_name</span><span class="si">:</span><span class="s2">20s</span><span class="si">}</span><span class="s2">] empty"</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expire</span> <span class="ow">in</span> <span class="n">kv_list</span><span class="p">:</span>
|
||||||
|
<span class="n">valid_until</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="k">if</span> <span class="n">expire</span><span class="p">:</span>
|
||||||
|
<span class="n">valid_until</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">expire</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
||||||
|
<span class="n">c_kv</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||||
|
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"[</span><span class="si">{</span><span class="n">ctx_name</span><span class="si">:</span><span class="s2">20s</span><span class="si">}</span><span class="s2">] </span><span class="si">{</span><span class="n">valid_until</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">key</span><span class="si">:</span><span class="s2">12</span><span class="si">}</span><span class="s2">"</span> <span class="sa">f</span><span class="s2">" --> (</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">) </span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2"> "</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Number of contexts: </span><span class="si">{</span><span class="n">c_ctx</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"number of key/value pairs: </span><span class="si">{</span><span class="n">c_kv</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">lines</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCache">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ExpireCache</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Abstract base class for the implementation of a key/value cache</span>
|
||||||
|
<span class="sd"> with expire date."""</span>
|
||||||
|
|
||||||
|
<span class="n">cfg</span><span class="p">:</span> <span class="n">ExpireCacheCfg</span>
|
||||||
|
|
||||||
|
<span class="n">hash_token</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"hash_token"</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCache.set">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.set">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Set *key* to *value*. To set a timeout on key use argument</span>
|
||||||
|
<span class="sd"> ``expire`` (in sec.). If expire is unset the default is taken from</span>
|
||||||
|
<span class="sd"> :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`. After the timeout has expired,</span>
|
||||||
|
<span class="sd"> the key will automatically be deleted.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The ``ctx`` argument specifies the context of the ``key``. A key is</span>
|
||||||
|
<span class="sd"> only unique in its context.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The concrete implementations of this abstraction determine how the</span>
|
||||||
|
<span class="sd"> context is mapped in the connected database. In SQL databases, for</span>
|
||||||
|
<span class="sd"> example, the context is a DB table or in a Key/Value DB it could be</span>
|
||||||
|
<span class="sd"> a prefix for the key.</span>
|
||||||
|
|
||||||
|
<span class="sd"> If the context is not specified (the default is ``None``) then a</span>
|
||||||
|
<span class="sd"> default context should be used, e.g. a default table for SQL databases</span>
|
||||||
|
<span class="sd"> or a default prefix in a Key/Value DB.</span>
|
||||||
|
<span class="sd"> """</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCache.get">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.get">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Return *value* of *key*. If key is unset, ``None`` is returned."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCache.maintenance">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.maintenance">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">truncate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Performs maintenance on the cache.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``force``:</span>
|
||||||
|
<span class="sd"> Maintenance should be carried out even if the maintenance interval has</span>
|
||||||
|
<span class="sd"> not yet been reached.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``truncate``:</span>
|
||||||
|
<span class="sd"> Truncate the entire cache, which is necessary, for example, if the</span>
|
||||||
|
<span class="sd"> password has changed.</span>
|
||||||
|
<span class="sd"> """</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCache.state">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.state">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ExpireCacheStats</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a :py:obj:`ExpireCacheStats`, which provides information</span>
|
||||||
|
<span class="sd"> about the status of the cache."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCache.build_cache">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.build_cache">[docs]</a>
|
||||||
|
<span class="nd">@staticmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">build_cache</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="n">ExpireCacheCfg</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"ExpireCacheSQLite"</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Factory to build a caching instance.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> Currently, only the SQLite adapter is available, but other database</span>
|
||||||
|
<span class="sd"> types could be implemented in the future, e.g. a Valkey (Redis)</span>
|
||||||
|
<span class="sd"> adapter.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="n">ExpireCacheSQLite</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCache.normalize_name">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.normalize_name">[docs]</a>
|
||||||
|
<span class="nd">@staticmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">normalize_name</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a normalized name that can be used as a file name or as a SQL</span>
|
||||||
|
<span class="sd"> table name (is used, for example, to normalize the context name)."""</span>
|
||||||
|
|
||||||
|
<span class="n">_valid</span> <span class="o">=</span> <span class="s2">"-_."</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_letters</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">name</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">_valid</span><span class="p">])</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bytes</span><span class="p">:</span>
|
||||||
|
<span class="n">dump</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">dump</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-></span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="n">obj</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">obj</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCache.secret_hash">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCache.secret_hash">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">secret_hash</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Creates a hash of the argument ``name``. The hash value is formed</span>
|
||||||
|
<span class="sd"> from the ``name`` combined with the :py:obj:`password</span>
|
||||||
|
<span class="sd"> <ExpireCacheCfg.password>`. Can be used, for example, to make the</span>
|
||||||
|
<span class="sd"> ``key`` stored in the DB unreadable for third parties."""</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">)</span>
|
||||||
|
<span class="n">m</span> <span class="o">=</span> <span class="n">hmac</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="n">name</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">password</span><span class="p">,</span> <span class="n">digestmod</span><span class="o">=</span><span class="s1">'sha256'</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">m</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ExpireCacheSQLite</span><span class="p">(</span><span class="n">sqlitedb</span><span class="o">.</span><span class="n">SQLiteAppl</span><span class="p">,</span> <span class="n">ExpireCache</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Cache that manages key/value pairs in a SQLite DB. The DB model in the</span>
|
||||||
|
<span class="sd"> SQLite DB is implemented in abstract class :py:obj:`SQLiteAppl</span>
|
||||||
|
<span class="sd"> <searx.sqlitedb.SQLiteAppl>`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The following configurations are required / supported:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - :py:obj:`ExpireCacheCfg.db_url`</span>
|
||||||
|
<span class="sd"> - :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`</span>
|
||||||
|
<span class="sd"> - :py:obj:`ExpireCacheCfg.MAINTENANCE_PERIOD`</span>
|
||||||
|
<span class="sd"> - :py:obj:`ExpireCacheCfg.MAINTENANCE_MODE`</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">DB_SCHEMA</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span>
|
||||||
|
|
||||||
|
<span class="c1"># The key/value tables will be created on demand by self.create_table</span>
|
||||||
|
<span class="n">DDL_CREATE_TABLES</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="n">CACHE_TABLE_PREFIX</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"CACHE-TABLE"</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">ExpireCacheCfg</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""An instance of the SQLite expire cache is build up from a</span>
|
||||||
|
<span class="sd"> :py:obj:`config <ExpireCacheCfg>`."""</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="p">:</span> <span class="n">ExpireCacheCfg</span> <span class="o">=</span> <span class="n">cfg</span>
|
||||||
|
<span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span> <span class="o">==</span> <span class="s2">":memory:"</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="s2">"don't use SQLite DB in :memory: in production!!"</span><span class="p">)</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite.init">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">:</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">Connection</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="n">ret_val</span> <span class="o">=</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">init</span><span class="p">(</span><span class="n">conn</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ret_val</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">new</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">password</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||||
|
<span class="n">old</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hash_token</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">old</span> <span class="o">!=</span> <span class="n">new</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">old</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"[</span><span class="si">%s</span><span class="s2">] hash token changed: truncate all cache tables"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">(</span><span class="n">force</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">truncate</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hash_token</span><span class="p">,</span> <span class="n">new</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite.maintenance">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.maintenance">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">truncate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">force</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_maintenance_time</span><span class="p">:</span>
|
||||||
|
<span class="c1"># log.debug("no maintenance required yet, next maintenance interval is in the future")</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="c1"># Prevent parallel DB maintenance cycles from other DB connections</span>
|
||||||
|
<span class="c1"># (e.g. in multi thread or process environments).</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">"LAST_MAINTENANCE"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span> <span class="c1"># hint: this (also) sets the m_time of the property!</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">truncate</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">truncate_tables</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="c1"># drop items by expire time stamp ..</span>
|
||||||
|
<span class="n">expire</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span>
|
||||||
|
|
||||||
|
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="sa">f</span><span class="s2">"DELETE FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2"> WHERE expire < ?"</span><span class="p">,</span> <span class="p">(</span><span class="n">expire</span><span class="p">,))</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"deleted </span><span class="si">%s</span><span class="s2"> keys from table </span><span class="si">%s</span><span class="s2"> (expire date reached)"</span><span class="p">,</span> <span class="n">res</span><span class="o">.</span><span class="n">rowcount</span><span class="p">,</span> <span class="n">table</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Vacuuming the WALs</span>
|
||||||
|
<span class="c1"># https://www.theunterminatedstring.com/sqlite-vacuuming/</span>
|
||||||
|
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"PRAGMA wal_checkpoint(TRUNCATE)"</span><span class="p">)</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite.create_table">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.create_table">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">create_table</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Create DB ``table`` if it has not yet been created, no recreates are</span>
|
||||||
|
<span class="sd"> initiated if the table already exists.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="n">table</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
|
||||||
|
<span class="c1"># log.debug("key/value table %s exists in DB (no need to recreate)", table)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"key/value table '</span><span class="si">%s</span><span class="s2">' NOT exists in DB -> create DB table .."</span><span class="p">,</span> <span class="n">table</span><span class="p">)</span>
|
||||||
|
<span class="n">sql_table</span> <span class="o">=</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||||
|
<span class="p">[</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"CREATE TABLE IF NOT EXISTS </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2"> ("</span><span class="p">,</span>
|
||||||
|
<span class="s2">" key TEXT,"</span><span class="p">,</span>
|
||||||
|
<span class="s2">" value BLOB,"</span><span class="p">,</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" expire INTEGER DEFAULT (strftime('%s', 'now') + </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAXHOLD_TIME</span><span class="si">}</span><span class="s2">),"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"PRIMARY KEY (key))"</span><span class="p">,</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">sql_index</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"CREATE INDEX IF NOT EXISTS index_expire_</span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2"> ON </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">(expire);"</span>
|
||||||
|
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql_table</span><span class="p">)</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql_index</span><span class="p">)</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">CACHE_TABLE_PREFIX</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> <span class="n">table</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="nd">@property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">table_names</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of key/value tables already created in the DB."""</span>
|
||||||
|
<span class="n">sql</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"SELECT value FROM properties WHERE name LIKE '</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">CACHE_TABLE_PREFIX</span><span class="si">}</span><span class="s2">%%'"</span>
|
||||||
|
<span class="n">rows</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span><span class="o">.</span><span class="n">fetchall</span><span class="p">()</span> <span class="ow">or</span> <span class="p">[]</span>
|
||||||
|
<span class="k">return</span> <span class="p">[</span><span class="n">r</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">truncate_tables</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">table_names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]):</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"truncate table: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">table_names</span><span class="p">))</span>
|
||||||
|
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="n">table_names</span><span class="p">:</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="sa">f</span><span class="s2">"DELETE FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="nd">@property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">next_maintenance_time</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns (unix epoch) time of the next maintenance."""</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAINTENANCE_PERIOD</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">m_time</span><span class="p">(</span><span class="s2">"LAST_MAINTENANCE"</span><span class="p">,</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()))</span>
|
||||||
|
|
||||||
|
<span class="c1"># implement ABC methods of ExpireCache</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite.set">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.set">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Set key/value in DB table given by argument ``ctx``. If expire is</span>
|
||||||
|
<span class="sd"> unset the default is taken from :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`.</span>
|
||||||
|
<span class="sd"> If ``ctx`` argument is ``None`` (the default), a table name is</span>
|
||||||
|
<span class="sd"> generated from the :py:obj:`ExpireCacheCfg.name`. If DB table does not</span>
|
||||||
|
<span class="sd"> exists, it will be created (on demand) by :py:obj:`self.create_table</span>
|
||||||
|
<span class="sd"> <ExpireCacheSQLite.create_table>`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">c</span><span class="p">,</span> <span class="n">err_msg_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_setmany</span><span class="p">([(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expire</span><span class="p">)],</span> <span class="n">ctx</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">c</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2"> -- </span><span class="si">%s</span><span class="s2">: key '</span><span class="si">%s</span><span class="s2">' updated or inserted (</span><span class="si">%s</span><span class="s2"> errors)"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">err_msg_list</span><span class="p">))</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">err_msg_list</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2"> -- </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">c</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite.setmany">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.setmany">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setmany</span><span class="p">(</span>
|
||||||
|
<span class="bp">self</span><span class="p">,</span>
|
||||||
|
<span class="n">opt_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">CacheRowType</span><span class="p">],</span>
|
||||||
|
<span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Efficient bootload of the cache from a list of options. The list</span>
|
||||||
|
<span class="sd"> contains tuples with the arguments described in</span>
|
||||||
|
<span class="sd"> :py:obj:`ExpireCacheSQLite.set`."""</span>
|
||||||
|
<span class="n">_start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
|
||||||
|
<span class="n">c</span><span class="p">,</span> <span class="n">err_msg_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_setmany</span><span class="p">(</span><span class="n">opt_list</span><span class="o">=</span><span class="n">opt_list</span><span class="p">,</span> <span class="n">ctx</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
|
||||||
|
<span class="n">_end</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">msg</span> <span class="ow">in</span> <span class="n">err_msg_list</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2"> -- </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||||
|
<span class="s2">"</span><span class="si">%s</span><span class="s2"> -- </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">/</span><span class="si">%s</span><span class="s2"> key/value pairs updated or inserted in </span><span class="si">%s</span><span class="s2"> sec (</span><span class="si">%s</span><span class="s2"> errors)"</span><span class="p">,</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
|
||||||
|
<span class="n">ctx</span><span class="p">,</span>
|
||||||
|
<span class="n">c</span><span class="p">,</span>
|
||||||
|
<span class="nb">len</span><span class="p">(</span><span class="n">opt_list</span><span class="p">),</span>
|
||||||
|
<span class="n">_end</span> <span class="o">-</span> <span class="n">_start</span><span class="p">,</span>
|
||||||
|
<span class="nb">len</span><span class="p">(</span><span class="n">err_msg_list</span><span class="p">),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">c</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_setmany</span><span class="p">(</span>
|
||||||
|
<span class="bp">self</span><span class="p">,</span>
|
||||||
|
<span class="n">opt_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">CacheRowType</span><span class="p">],</span>
|
||||||
|
<span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
|
||||||
|
|
||||||
|
<span class="n">table</span> <span class="o">=</span> <span class="n">ctx</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">table_name</span> <span class="o">=</span> <span class="n">table</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">table_name</span><span class="p">:</span>
|
||||||
|
<span class="n">table_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalize_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">create_table</span><span class="p">(</span><span class="n">table_name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">sql_str</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"INSERT INTO </span><span class="si">{</span><span class="n">table_name</span><span class="si">}</span><span class="s2"> (key, value, expire) VALUES (?, ?, ?)"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" ON CONFLICT DO "</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"UPDATE SET value=?, expire=?"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">sql_rows</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span>
|
||||||
|
<span class="nb">tuple</span><span class="p">[</span>
|
||||||
|
<span class="nb">str</span><span class="p">,</span> <span class="c1"># key</span>
|
||||||
|
<span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="c1"># value</span>
|
||||||
|
<span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="c1"># expire</span>
|
||||||
|
<span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="c1"># value</span>
|
||||||
|
<span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="c1"># expire</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">err_msg_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">_val</span><span class="p">,</span> <span class="n">expire</span> <span class="ow">in</span> <span class="n">opt_list</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">value</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">serialize</span><span class="p">(</span><span class="n">value</span><span class="o">=</span><span class="n">_val</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAX_VALUE_LEN</span><span class="p">:</span>
|
||||||
|
<span class="n">err_msg_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">.key='</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">' - serialized value too big to cache (len: </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="si">}</span><span class="s2">) "</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">expire</span><span class="p">:</span>
|
||||||
|
<span class="n">expire</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAXHOLD_TIME</span>
|
||||||
|
<span class="n">expire</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">+</span> <span class="n">expire</span>
|
||||||
|
|
||||||
|
<span class="c1"># positional arguments of the INSERT INTO statement</span>
|
||||||
|
<span class="n">sql_args</span> <span class="o">=</span> <span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expire</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">expire</span><span class="p">)</span>
|
||||||
|
<span class="n">sql_rows</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">sql_args</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">sql_rows</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="mi">0</span><span class="p">,</span> <span class="n">err_msg_list</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">table</span><span class="p">:</span>
|
||||||
|
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">executemany</span><span class="p">(</span><span class="n">sql_str</span><span class="p">,</span> <span class="n">sql_rows</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">executemany</span><span class="p">(</span><span class="n">sql_str</span><span class="p">,</span> <span class="n">sql_rows</span><span class="p">)</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">sql_rows</span><span class="p">),</span> <span class="n">err_msg_list</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite.get">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.get">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get value of ``key`` from table given by argument ``ctx``. If</span>
|
||||||
|
<span class="sd"> ``ctx`` argument is ``None`` (the default), a table name is generated</span>
|
||||||
|
<span class="sd"> from the :py:obj:`ExpireCacheCfg.name`. If ``key`` not exists (in</span>
|
||||||
|
<span class="sd"> table), the ``default`` value is returned.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">table</span> <span class="o">=</span> <span class="n">ctx</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">table</span><span class="p">:</span>
|
||||||
|
<span class="n">table</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalize_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">table</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">default</span>
|
||||||
|
|
||||||
|
<span class="n">sql</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"SELECT value FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2"> WHERE key = ?"</span>
|
||||||
|
<span class="n">row</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="p">(</span><span class="n">key</span><span class="p">,))</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">row</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">default</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">row</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite.pairs">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.pairs">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">pairs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ctx</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Iterator</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Iterate over key/value pairs from table given by argument ``ctx``.</span>
|
||||||
|
<span class="sd"> If ``ctx`` argument is ``None`` (the default), a table name is</span>
|
||||||
|
<span class="sd"> generated from the :py:obj:`ExpireCacheCfg.name`."""</span>
|
||||||
|
<span class="n">table</span> <span class="o">=</span> <span class="n">ctx</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">table</span><span class="p">:</span>
|
||||||
|
<span class="n">table</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalize_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">table</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="sa">f</span><span class="s2">"SELECT key, value FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">"</span><span class="p">):</span>
|
||||||
|
<span class="k">yield</span> <span class="n">row</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">row</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ExpireCacheSQLite.state">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.cache.html#searx.cache.ExpireCacheSQLite.state">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">ExpireCacheStats</span><span class="p">:</span>
|
||||||
|
<span class="n">cached_items</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="n">CacheRowType</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="k">for</span> <span class="n">table</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">table_names</span><span class="p">:</span>
|
||||||
|
<span class="n">cached_items</span><span class="p">[</span><span class="n">table</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="sa">f</span><span class="s2">"SELECT key, value, expire FROM </span><span class="si">{</span><span class="n">table</span><span class="si">}</span><span class="s2">"</span><span class="p">):</span>
|
||||||
|
<span class="n">cached_items</span><span class="p">[</span><span class="n">table</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">row</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">row</span><span class="p">[</span><span class="mi">1</span><span class="p">]),</span> <span class="n">row</span><span class="p">[</span><span class="mi">2</span><span class="p">]))</span>
|
||||||
|
<span class="k">return</span> <span class="n">ExpireCacheStats</span><span class="p">(</span><span class="n">cached_items</span><span class="o">=</span><span class="n">cached_items</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
482
_modules/searx/enginelib.html
Normal file
@@ -0,0 +1,482 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.enginelib — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.enginelib</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.enginelib</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Implementations of the framework for the SearXNG engines.</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`searx.enginelib.EngineCache`</span>
|
||||||
|
<span class="sd">- :py:obj:`searx.enginelib.Engine`</span>
|
||||||
|
<span class="sd">- :py:obj:`searx.enginelib.traits`</span>
|
||||||
|
|
||||||
|
<span class="sd">There is a command line for developer purposes and for deeper analysis. Here is</span>
|
||||||
|
<span class="sd">an example in which the command line is called in the development environment::</span>
|
||||||
|
|
||||||
|
<span class="sd"> $ ./manage pyenv.cmd bash --norc --noprofile</span>
|
||||||
|
<span class="sd"> (py3) python -m searx.enginelib --help</span>
|
||||||
|
|
||||||
|
<span class="sd">.. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> The long term goal is to modularize all implementations of the engine</span>
|
||||||
|
<span class="sd"> framework here in this Python package. ToDo:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - move implementations of the :ref:`searx.engines loader` to a new module in</span>
|
||||||
|
<span class="sd"> the :py:obj:`searx.enginelib` namespace.</span>
|
||||||
|
|
||||||
|
<span class="sd">-----</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"EngineCache"</span><span class="p">,</span> <span class="s2">"Engine"</span><span class="p">,</span> <span class="s2">"ENGINES_CACHE"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typer</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">..cache</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpireCacheSQLite</span><span class="p">,</span> <span class="n">ExpireCacheCfg</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">traits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OfflineParamTypes</span><span class="p">,</span> <span class="n">OnlineParamTypes</span>
|
||||||
|
|
||||||
|
<span class="n">ENGINES_CACHE</span><span class="p">:</span> <span class="n">ExpireCacheSQLite</span> <span class="o">=</span> <span class="n">ExpireCacheSQLite</span><span class="o">.</span><span class="n">build_cache</span><span class="p">(</span>
|
||||||
|
<span class="n">ExpireCacheCfg</span><span class="p">(</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="s2">"ENGINES_CACHE"</span><span class="p">,</span>
|
||||||
|
<span class="n">MAXHOLD_TIME</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span> <span class="c1"># 7 days</span>
|
||||||
|
<span class="n">MAINTENANCE_PERIOD</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span><span class="p">,</span> <span class="c1"># 2h</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="sd">"""Global :py:obj:`searx.cache.ExpireCacheSQLite` instance where the cached</span>
|
||||||
|
<span class="sd">values from all engines are stored. The `MAXHOLD_TIME` is 7 days and the</span>
|
||||||
|
<span class="sd">`MAINTENANCE_PERIOD` is set to two hours."""</span>
|
||||||
|
|
||||||
|
<span class="n">app</span> <span class="o">=</span> <span class="n">typer</span><span class="o">.</span><span class="n">Typer</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">():</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Show state for the caches of the engines."""</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="s2">"cache tables and key/values"</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"="</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">title</span><span class="p">))</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span><span class="o">.</span><span class="n">report</span><span class="p">())</span>
|
||||||
|
<span class="nb">print</span><span class="p">()</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"properties of </span><span class="si">{</span><span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"="</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">title</span><span class="p">))</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">properties</span><span class="p">))</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Carry out maintenance on cache of the engines."""</span>
|
||||||
|
<span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">maintenance</span><span class="p">(</span><span class="n">force</span><span class="o">=</span><span class="n">force</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineCache">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.EngineCache">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">EngineCache</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Persistent (SQLite) key/value cache that deletes its values again after</span>
|
||||||
|
<span class="sd"> ``expire`` seconds (default/max: :py:obj:`MAXHOLD_TIME</span>
|
||||||
|
<span class="sd"> <searx.cache.ExpireCacheCfg.MAXHOLD_TIME>`). This class is a wrapper around</span>
|
||||||
|
<span class="sd"> :py:obj:`ENGINES_CACHE` (:py:obj:`ExpireCacheSQLite</span>
|
||||||
|
<span class="sd"> <searx.cache.ExpireCacheSQLite>`).</span>
|
||||||
|
|
||||||
|
<span class="sd"> In the :origin:`searx/engines/demo_offline.py` engine you can find an</span>
|
||||||
|
<span class="sd"> exemplary implementation of such a cache other examples are implemented</span>
|
||||||
|
<span class="sd"> in:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - :origin:`searx/engines/radio_browser.py`</span>
|
||||||
|
<span class="sd"> - :origin:`searx/engines/soundcloud.py`</span>
|
||||||
|
<span class="sd"> - :origin:`searx/engines/startpage.py`</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> from searx.enginelib import EngineCache</span>
|
||||||
|
<span class="sd"> CACHE: EngineCache</span>
|
||||||
|
|
||||||
|
<span class="sd"> def init(engine_settings):</span>
|
||||||
|
<span class="sd"> global CACHE</span>
|
||||||
|
<span class="sd"> CACHE = EngineCache(engine_settings["name"])</span>
|
||||||
|
|
||||||
|
<span class="sd"> def request(query, params):</span>
|
||||||
|
<span class="sd"> token = CACHE.get(key="token")</span>
|
||||||
|
<span class="sd"> if token is None:</span>
|
||||||
|
<span class="sd"> token = get_token()</span>
|
||||||
|
<span class="sd"> # cache token of this engine for 1h</span>
|
||||||
|
<span class="sd"> CACHE.set(key="token", value=token, expire=3600)</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
|
||||||
|
<span class="sd"> For introspection of the DB, jump into developer environment and run command to</span>
|
||||||
|
<span class="sd"> show cache state::</span>
|
||||||
|
|
||||||
|
<span class="sd"> $ ./manage pyenv.cmd bash --norc --noprofile</span>
|
||||||
|
<span class="sd"> (py3) python -m searx.enginelib cache state</span>
|
||||||
|
|
||||||
|
<span class="sd"> cache tables and key/values</span>
|
||||||
|
<span class="sd"> ===========================</span>
|
||||||
|
<span class="sd"> [demo_offline ] 2025-04-22 11:32:50 count --> (int) 4</span>
|
||||||
|
<span class="sd"> [startpage ] 2025-04-22 12:32:30 SC_CODE --> (str) fSOBnhEMlDfE20</span>
|
||||||
|
<span class="sd"> [duckduckgo ] 2025-04-22 12:32:31 4dff493e.... --> (str) 4-128634958369380006627592672385352473325</span>
|
||||||
|
<span class="sd"> [duckduckgo ] 2025-04-22 12:40:06 3e2583e2.... --> (str) 4-263126175288871260472289814259666848451</span>
|
||||||
|
<span class="sd"> [radio_browser ] 2025-04-23 11:33:08 servers --> (list) ['https://de2.api.radio-browser.info', ...]</span>
|
||||||
|
<span class="sd"> [soundcloud ] 2025-04-29 11:40:06 guest_client_id --> (str) EjkRJG0BLNEZquRiPZYdNtJdyGtTuHdp</span>
|
||||||
|
<span class="sd"> [wolframalpha ] 2025-04-22 12:40:06 code --> (str) 5aa79f86205ad26188e0e26e28fb7ae7</span>
|
||||||
|
<span class="sd"> number of tables: 6</span>
|
||||||
|
<span class="sd"> number of key/value pairs: 7</span>
|
||||||
|
|
||||||
|
<span class="sd"> In the "cache tables and key/values" section, the table name (engine name) is at</span>
|
||||||
|
<span class="sd"> first position on the second there is the calculated expire date and on the</span>
|
||||||
|
<span class="sd"> third and fourth position the key/value is shown.</span>
|
||||||
|
|
||||||
|
<span class="sd"> About duckduckgo: The *vqd coode* of ddg depends on the query term and therefore</span>
|
||||||
|
<span class="sd"> the key is a hash value of the query term (to not to store the raw query term).</span>
|
||||||
|
|
||||||
|
<span class="sd"> In the "properties of ENGINES_CACHE" section all properties of the SQLiteAppl /</span>
|
||||||
|
<span class="sd"> ExpireCache and their last modification date are shown::</span>
|
||||||
|
|
||||||
|
<span class="sd"> properties of ENGINES_CACHE</span>
|
||||||
|
<span class="sd"> ===========================</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:32:27] DB_SCHEMA : 1</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:32:27] LAST_MAINTENANCE :</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:32:27] crypt_hash : ca612e3566fdfd7cf7efe...</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:32:30] CACHE-TABLE--demo_offline: demo_offline</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:32:30] CACHE-TABLE--startpage: startpage</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:32:31] CACHE-TABLE--duckduckgo: duckduckgo</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:33:08] CACHE-TABLE--radio_browser: radio_browser</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:40:06] CACHE-TABLE--soundcloud: soundcloud</span>
|
||||||
|
<span class="sd"> [last modified: 2025-04-22 11:40:06] CACHE-TABLE--wolframalpha: wolframalpha</span>
|
||||||
|
|
||||||
|
<span class="sd"> These properties provide information about the state of the ExpireCache and</span>
|
||||||
|
<span class="sd"> control the behavior. For example, the maintenance intervals are controlled by</span>
|
||||||
|
<span class="sd"> the last modification date of the LAST_MAINTENANCE property and the hash value</span>
|
||||||
|
<span class="sd"> of the password can be used to detect whether the password has been changed (in</span>
|
||||||
|
<span class="sd"> this case the DB entries can no longer be decrypted and the entire cache must be</span>
|
||||||
|
<span class="sd"> discarded).</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">expire</span> <span class="ow">or</span> <span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAXHOLD_TIME</span>
|
||||||
|
<span class="n">_valid</span> <span class="o">=</span> <span class="s2">"-_."</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_letters</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">c</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">_valid</span> <span class="k">else</span> <span class="s2">"_"</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">engine_name</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span>
|
||||||
|
<span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
|
||||||
|
<span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span>
|
||||||
|
<span class="n">expire</span><span class="o">=</span><span class="n">expire</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">expire</span><span class="p">,</span>
|
||||||
|
<span class="n">ctx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">,</span> <span class="n">ctx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">table_name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">secret_hash</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">ENGINES_CACHE</span><span class="o">.</span><span class="n">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Engine">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Engine</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Class of engine instances build from YAML settings.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Further documentation see :ref:`general engine configuration`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> This class is currently never initialized and only used for type hinting.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span>
|
||||||
|
|
||||||
|
<span class="c1"># Common options in the engine module</span>
|
||||||
|
|
||||||
|
<span class="n">engine_type</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Type of the engine (:ref:`searx.search.processors`)"""</span>
|
||||||
|
|
||||||
|
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Engine supports multiple pages."""</span>
|
||||||
|
|
||||||
|
<span class="n">max_page</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""If the engine supports paging, then this is the value for the last page</span>
|
||||||
|
<span class="sd"> that is still supported. ``0`` means unlimited numbers of pages."""</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Engine supports search time range."""</span>
|
||||||
|
|
||||||
|
<span class="n">safesearch</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Engine supports SafeSearch"""</span>
|
||||||
|
|
||||||
|
<span class="n">language_support</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Engine supports languages (locales) search."""</span>
|
||||||
|
|
||||||
|
<span class="n">language</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""For an engine, when there is ``language: ...`` in the YAML settings the engine</span>
|
||||||
|
<span class="sd"> does support only this one language:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: google french</span>
|
||||||
|
<span class="sd"> engine: google</span>
|
||||||
|
<span class="sd"> language: fr</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">region</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""For an engine, when there is ``region: ...`` in the YAML settings the engine</span>
|
||||||
|
<span class="sd"> does support only this one region::</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: google belgium</span>
|
||||||
|
<span class="sd"> engine: google</span>
|
||||||
|
<span class="sd"> region: fr-BE</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">fetch_traits</span><span class="p">:</span> <span class="s2">"Callable[[EngineTraits, bool], None]"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Function to to fetch engine's traits from origin."""</span>
|
||||||
|
|
||||||
|
<span class="n">traits</span><span class="p">:</span> <span class="s2">"traits.EngineTraits"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Traits of the engine."""</span>
|
||||||
|
|
||||||
|
<span class="c1"># settings.yml</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Specifies to which :ref:`engine categories` the engine should be added."""</span>
|
||||||
|
|
||||||
|
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name that will be used across SearXNG to define this engine. In settings, on</span>
|
||||||
|
<span class="sd"> the result page .."""</span>
|
||||||
|
|
||||||
|
<span class="n">engine</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the python file used to handle requests and responses to and from</span>
|
||||||
|
<span class="sd"> this search engine (file name from :origin:`searx/engines` without</span>
|
||||||
|
<span class="sd"> ``.py``)."""</span>
|
||||||
|
|
||||||
|
<span class="n">enable_http</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Enable HTTP (by default only HTTPS is enabled)."""</span>
|
||||||
|
|
||||||
|
<span class="n">shortcut</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Code used to execute bang requests (``!foo``)"""</span>
|
||||||
|
|
||||||
|
<span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Specific timeout for search-engine."""</span>
|
||||||
|
|
||||||
|
<span class="n">display_error_messages</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Display error messages on the web UI."""</span>
|
||||||
|
|
||||||
|
<span class="n">proxies</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Set proxies for a specific engine (YAML):</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> proxies :</span>
|
||||||
|
<span class="sd"> http: socks5://proxy:port</span>
|
||||||
|
<span class="sd"> https: socks5://proxy:port</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">disabled</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""To disable by default the engine, but not deleting it. It will allow the</span>
|
||||||
|
<span class="sd"> user to manually activate it in the settings."""</span>
|
||||||
|
|
||||||
|
<span class="n">inactive</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Remove the engine from the settings (*disabled & removed*)."""</span>
|
||||||
|
|
||||||
|
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Additional fields describing the engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> about:</span>
|
||||||
|
<span class="sd"> website: https://example.com</span>
|
||||||
|
<span class="sd"> wikidata_id: Q306656</span>
|
||||||
|
<span class="sd"> official_api_documentation: https://example.com/api-doc</span>
|
||||||
|
<span class="sd"> use_official_api: true</span>
|
||||||
|
<span class="sd"> require_api_key: true</span>
|
||||||
|
<span class="sd"> results: HTML</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">using_tor_proxy</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Using tor proxy (``true``) or not (``false``) for this engine."""</span>
|
||||||
|
|
||||||
|
<span class="n">send_accept_language_header</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""When this option is activated, the language (locale) that is selected by</span>
|
||||||
|
<span class="sd"> the user is used to build and send a ``Accept-Language`` header in the</span>
|
||||||
|
<span class="sd"> request to the origin search engine."""</span>
|
||||||
|
|
||||||
|
<span class="n">tokens</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A list of secret tokens to make this engine *private*, more details see</span>
|
||||||
|
<span class="sd"> :ref:`private engines`."""</span>
|
||||||
|
|
||||||
|
<span class="n">weight</span><span class="p">:</span> <span class="nb">int</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Weighting of the results of this engine (:ref:`weight <settings engines>`)."""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Engine.setup">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Dynamic setup of the engine settings.</span>
|
||||||
|
|
||||||
|
<span class="sd"> With this method, the engine's setup is carried out. For example, to</span>
|
||||||
|
<span class="sd"> check or dynamically adapt the values handed over in the parameter</span>
|
||||||
|
<span class="sd"> ``engine_settings``. The return value (True/False) indicates whether</span>
|
||||||
|
<span class="sd"> the setup was successful and the engine can be built or rejected.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The method is optional and is called synchronously as part of the</span>
|
||||||
|
<span class="sd"> initialization of the service and is therefore only suitable for simple</span>
|
||||||
|
<span class="sd"> (local) exams/changes at the engine setting. The :py:obj:`Engine.init`</span>
|
||||||
|
<span class="sd"> method must be used for longer tasks in which values of a remote must be</span>
|
||||||
|
<span class="sd"> determined, for example.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Engine.init">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The method is optional and asynchronous (in a thread). It is suitable,</span>
|
||||||
|
<span class="sd"> for example, for setting up a cache (for the engine) or for querying</span>
|
||||||
|
<span class="sd"> values (required by the engine) from a remote.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Whether the initialization was successful can be indicated by the return</span>
|
||||||
|
<span class="sd"> value ``True`` or even ``False``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - If no return value is given from this init method (``None``), this is</span>
|
||||||
|
<span class="sd"> equivalent to ``True``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - If an exception is thrown as part of the initialization, this is</span>
|
||||||
|
<span class="sd"> equivalent to ``False``.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Engine.search">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.search">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OfflineParamTypes"</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"EngineResults"</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Search method of the ``offline`` engines"""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Engine.request">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.request">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParamTypes"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Method to build the parameters for the request of an ``online``</span>
|
||||||
|
<span class="sd"> engine."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Engine.response">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/enginelib.html#searx.enginelib.Engine.response">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"EngineResults"</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Method to parse the response of an ``online`` engine."""</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
410
_modules/searx/enginelib/traits.html
Normal file
@@ -0,0 +1,410 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.enginelib.traits — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../enginelib.html" accesskey="U">searx.enginelib</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.enginelib.traits</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.enginelib.traits</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Engine's traits are fetched from the origin engines and stored in a JSON file</span>
|
||||||
|
<span class="sd">in the *data folder*. Most often traits are languages and region codes and</span>
|
||||||
|
<span class="sd">their mapping from SearXNG's representation to the representation in the origin</span>
|
||||||
|
<span class="sd">search engine. For new traits new properties can be added to the class</span>
|
||||||
|
<span class="sd">:py:class:`EngineTraits`.</span>
|
||||||
|
|
||||||
|
<span class="sd">To load traits from the persistence :py:obj:`EngineTraitsMap.from_data` can be</span>
|
||||||
|
<span class="sd">used.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">dataclasses</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">types</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">locales</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">data_dir</span><span class="p">,</span> <span class="n">ENGINE_TRAITS</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Engine</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraitsEncoder">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsEncoder">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">EngineTraitsEncoder</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">JSONEncoder</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Encodes :class:`EngineTraits` to a serializable object, see</span>
|
||||||
|
<span class="sd"> :class:`json.JSONEncoder`."""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraitsEncoder.default">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsEncoder.default">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">default</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">o</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Return dictionary of a :class:`EngineTraits` object."""</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">o</span><span class="o">.</span><span class="vm">__dict__</span>
|
||||||
|
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">default</span><span class="p">(</span><span class="n">o</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits">[docs]</a>
|
||||||
|
<span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">EngineTraits</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The class is intended to be instantiated for each engine."""</span>
|
||||||
|
|
||||||
|
<span class="n">regions</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">dataclasses</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">dict</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Maps SearXNG's internal representation of a region to the one of the engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> SearXNG's internal representation can be parsed by babel and the value is</span>
|
||||||
|
<span class="sd"> send to the engine:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> regions ={</span>
|
||||||
|
<span class="sd"> 'fr-BE' : <engine's region name>,</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
|
||||||
|
<span class="sd"> for key, egnine_region regions.items():</span>
|
||||||
|
<span class="sd"> searxng_region = babel.Locale.parse(key, sep='-')</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">languages</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">dataclasses</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">dict</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Maps SearXNG's internal representation of a language to the one of the engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> SearXNG's internal representation can be parsed by babel and the value is</span>
|
||||||
|
<span class="sd"> send to the engine:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> languages = {</span>
|
||||||
|
<span class="sd"> 'ca' : <engine's language name>,</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
|
||||||
|
<span class="sd"> for key, egnine_lang in languages.items():</span>
|
||||||
|
<span class="sd"> searxng_lang = babel.Locale.parse(key)</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">all_locale</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""To which locale value SearXNG's ``all`` language is mapped (shown a "Default</span>
|
||||||
|
<span class="sd"> language").</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">data_type</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s1">'traits_v1'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'traits_v1'</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Data type, default is 'traits_v1'.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">custom</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">dataclasses</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">dict</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A place to store engine's custom traits, not related to the SearXNG core.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraits.get_language">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.get_language">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_language</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Return engine's language string that *best fits* to SearXNG's locale.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param searxng_locale: SearXNG's internal representation of locale</span>
|
||||||
|
<span class="sd"> selected by the user.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param default: engine's default language</span>
|
||||||
|
|
||||||
|
<span class="sd"> The *best fits* rules are implemented in</span>
|
||||||
|
<span class="sd"> :py:obj:`searx.locales.get_engine_locale`. Except for the special value ``all``</span>
|
||||||
|
<span class="sd"> which is determined from :py:obj:`EngineTraits.all_locale`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="n">searxng_locale</span> <span class="o">==</span> <span class="s1">'all'</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_locale</span>
|
||||||
|
<span class="k">return</span> <span class="n">locales</span><span class="o">.</span><span class="n">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">languages</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraits.get_region">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.get_region">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_region</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Return engine's region string that best fits to SearXNG's locale.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param searxng_locale: SearXNG's internal representation of locale</span>
|
||||||
|
<span class="sd"> selected by the user.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param default: engine's default region</span>
|
||||||
|
|
||||||
|
<span class="sd"> The *best fits* rules are implemented in</span>
|
||||||
|
<span class="sd"> :py:obj:`searx.locales.get_engine_locale`. Except for the special value ``all``</span>
|
||||||
|
<span class="sd"> which is determined from :py:obj:`EngineTraits.all_locale`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="n">searxng_locale</span> <span class="o">==</span> <span class="s1">'all'</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_locale</span>
|
||||||
|
<span class="k">return</span> <span class="n">locales</span><span class="o">.</span><span class="n">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">regions</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraits.is_locale_supported">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.is_locale_supported">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_locale_supported</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A *locale* (SearXNG's internal representation) is considered to be</span>
|
||||||
|
<span class="sd"> supported by the engine if the *region* or the *language* is supported</span>
|
||||||
|
<span class="sd"> by the engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> For verification the functions :py:func:`EngineTraits.get_region` and</span>
|
||||||
|
<span class="sd"> :py:func:`EngineTraits.get_language` are used.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_type</span> <span class="o">==</span> <span class="s1">'traits_v1'</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'engine traits of type </span><span class="si">%s</span><span class="s1"> is unknown'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_type</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraits.copy">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.copy">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Create a copy of the dataclass object."""</span>
|
||||||
|
<span class="k">return</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">dataclasses</span><span class="o">.</span><span class="n">asdict</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span> <span class="c1"># type: ignore</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraits.fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.fetch_traits">[docs]</a>
|
||||||
|
<span class="nd">@classmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"EngineTraits | None"</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Call a function ``fetch_traits(engine_traits)`` from engines namespace to fetch</span>
|
||||||
|
<span class="sd"> and set properties from the origin engine in the object ``engine_traits``. If</span>
|
||||||
|
<span class="sd"> function does not exists, ``None`` is returned.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">fetch_traits</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'fetch_traits'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">engine_traits</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">fetch_traits</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">()</span>
|
||||||
|
<span class="n">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">engine_traits</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraits.set_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraits.set_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set_traits</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Set traits from self object in a :py:obj:`.Engine` namespace.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param engine: engine instance build by :py:func:`searx.engines.load_engine`</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_type</span> <span class="o">==</span> <span class="s1">'traits_v1'</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">_set_traits_v1</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">'engine traits of type </span><span class="si">%s</span><span class="s1"> is unknown'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">data_type</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_set_traits_v1</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">):</span>
|
||||||
|
<span class="c1"># For an engine, when there is `language: ...` in the YAML settings the engine</span>
|
||||||
|
<span class="c1"># does support only this one language (region)::</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># - name: google italian</span>
|
||||||
|
<span class="c1"># engine: google</span>
|
||||||
|
<span class="c1"># language: it</span>
|
||||||
|
<span class="c1"># region: it-IT</span>
|
||||||
|
|
||||||
|
<span class="n">traits</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">_msg</span> <span class="o">=</span> <span class="s2">"settings.yml - engine: '</span><span class="si">%s</span><span class="s2">' / </span><span class="si">%s</span><span class="s2">: '</span><span class="si">%s</span><span class="s2">' not supported"</span>
|
||||||
|
|
||||||
|
<span class="n">languages</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">languages</span>
|
||||||
|
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'language'</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">language</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">_msg</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">'language'</span><span class="p">,</span> <span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">))</span>
|
||||||
|
<span class="n">traits</span><span class="o">.</span><span class="n">languages</span> <span class="o">=</span> <span class="p">{</span><span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">:</span> <span class="n">languages</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">language</span><span class="p">]}</span>
|
||||||
|
|
||||||
|
<span class="n">regions</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">regions</span>
|
||||||
|
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'region'</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">region</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">regions</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">_msg</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="s1">'region'</span><span class="p">,</span> <span class="n">engine</span><span class="o">.</span><span class="n">region</span><span class="p">))</span>
|
||||||
|
<span class="n">traits</span><span class="o">.</span><span class="n">regions</span> <span class="o">=</span> <span class="p">{</span><span class="n">engine</span><span class="o">.</span><span class="n">region</span><span class="p">:</span> <span class="n">regions</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">region</span><span class="p">]}</span>
|
||||||
|
|
||||||
|
<span class="n">engine</span><span class="o">.</span><span class="n">language_support</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="n">traits</span><span class="o">.</span><span class="n">languages</span> <span class="ow">or</span> <span class="n">traits</span><span class="o">.</span><span class="n">regions</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="c1"># set the copied & modified traits in engine's namespace</span>
|
||||||
|
<span class="n">engine</span><span class="o">.</span><span class="n">traits</span> <span class="o">=</span> <span class="n">traits</span> <span class="c1"># pyright: ignore[reportAttributeAccessIssue]</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraitsMap">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsMap">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">EngineTraitsMap</span><span class="p">(</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">EngineTraits</span><span class="p">]):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A python dictionary to map :class:`EngineTraits` by engine name."""</span>
|
||||||
|
|
||||||
|
<span class="n">ENGINE_TRAITS_FILE</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span> <span class="o">=</span> <span class="p">(</span><span class="n">data_dir</span> <span class="o">/</span> <span class="s1">'engine_traits.json'</span><span class="p">)</span><span class="o">.</span><span class="n">resolve</span><span class="p">()</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""File with persistence of the :py:obj:`EngineTraitsMap`."""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraitsMap.save_data">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsMap.save_data">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">save_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Store EngineTraitsMap in in file :py:obj:`self.ENGINE_TRAITS_FILE`"""</span>
|
||||||
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ENGINE_TRAITS_FILE</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||||
|
<span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">sort_keys</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="bp">cls</span><span class="o">=</span><span class="n">EngineTraitsEncoder</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraitsMap.from_data">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsMap.from_data">[docs]</a>
|
||||||
|
<span class="nd">@classmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">from_data</span><span class="p">(</span><span class="bp">cls</span><span class="p">)</span> <span class="o">-></span> <span class="s1">'EngineTraitsMap'</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Instantiate :class:`EngineTraitsMap` object from :py:obj:`ENGINE_TRAITS`"""</span>
|
||||||
|
<span class="n">obj</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">ENGINE_TRAITS</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">obj</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">v</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">obj</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="nd">@classmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">log</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="kc">None</span><span class="p">])</span> <span class="o">-></span> <span class="s1">'EngineTraitsMap'</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">engines</span> <span class="c1"># pylint: disable=cyclic-import, import-outside-toplevel</span>
|
||||||
|
|
||||||
|
<span class="n">names</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="p">)</span>
|
||||||
|
<span class="n">names</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
|
||||||
|
<span class="n">obj</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">engine_name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
|
||||||
|
<span class="n">engine</span><span class="p">:</span> <span class="n">Engine</span> <span class="o">|</span> <span class="n">types</span><span class="o">.</span><span class="n">ModuleType</span> <span class="o">=</span> <span class="n">engines</span><span class="o">.</span><span class="n">engines</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span>
|
||||||
|
<span class="n">traits</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># pylint: disable=broad-exception-caught</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">traits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="o">.</span><span class="n">fetch_traits</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="p">(</span><span class="s2">"FATAL: while fetch_traits </span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="n">exc</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'FORCE'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'on'</span><span class="p">,</span> <span class="s1">'true'</span><span class="p">,</span> <span class="s1">'1'</span><span class="p">]:</span>
|
||||||
|
<span class="k">raise</span>
|
||||||
|
<span class="n">v</span> <span class="o">=</span> <span class="n">ENGINE_TRAITS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">engine_name</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">v</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="p">(</span><span class="s2">"FORCE: re-use old values from fetch_traits - ENGINE_TRAITS[</span><span class="si">%s</span><span class="s2">]"</span> <span class="o">%</span> <span class="n">engine_name</span><span class="p">)</span>
|
||||||
|
<span class="n">traits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">v</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">traits</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="p">(</span><span class="s2">"</span><span class="si">%-20s</span><span class="s2">: SearXNG languages --> </span><span class="si">%s</span><span class="s2"> "</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">traits</span><span class="o">.</span><span class="n">languages</span><span class="p">)))</span>
|
||||||
|
<span class="n">log</span><span class="p">(</span><span class="s2">"</span><span class="si">%-20s</span><span class="s2">: SearXNG regions --> </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">engine_name</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">traits</span><span class="o">.</span><span class="n">regions</span><span class="p">)))</span>
|
||||||
|
<span class="n">obj</span><span class="p">[</span><span class="n">engine_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">traits</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">obj</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineTraitsMap.set_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/enginelib.html#searx.enginelib.traits.EngineTraitsMap.set_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set_traits</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Set traits in a :py:obj:`Engine` namespace.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param engine: engine instance build by :py:func:`searx.engines.load_engine`</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="n">data_type</span><span class="o">=</span><span class="s1">'traits_v1'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
||||||
|
<span class="n">engine_traits</span> <span class="o">=</span> <span class="bp">self</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">engine</span><span class="o">.</span><span class="n">engine</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
||||||
|
<span class="c1"># The key of the dictionary traits_map is the *engine name*</span>
|
||||||
|
<span class="c1"># configured in settings.xml. When multiple engines are configured</span>
|
||||||
|
<span class="c1"># in settings.yml to use the same origin engine (python module)</span>
|
||||||
|
<span class="c1"># these additional engines can use the languages from the origin</span>
|
||||||
|
<span class="c1"># engine. For this use the configured ``engine: ...`` from</span>
|
||||||
|
<span class="c1"># settings.yml</span>
|
||||||
|
<span class="n">engine_traits</span> <span class="o">=</span> <span class="bp">self</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">engine</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">set_traits</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../enginelib.html">searx.enginelib</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
399
_modules/searx/engines.html
Normal file
@@ -0,0 +1,399 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Load and initialize the ``engines``, see :py:func:`load_engines` and register</span>
|
||||||
|
<span class="sd">:py:obj:`engine_shortcuts`.</span>
|
||||||
|
|
||||||
|
<span class="sd">usage::</span>
|
||||||
|
|
||||||
|
<span class="sd"> load_engines( settings['engines'] )</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">copy</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">os.path</span><span class="w"> </span><span class="kn">import</span> <span class="n">realpath</span><span class="p">,</span> <span class="n">dirname</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">types</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">inspect</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span><span class="p">,</span> <span class="n">settings</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_module</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Engine</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'engines'</span><span class="p">)</span>
|
||||||
|
<span class="n">ENGINE_DIR</span> <span class="o">=</span> <span class="n">dirname</span><span class="p">(</span><span class="n">realpath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># Defaults for the namespace of an engine module, see load_engine()</span>
|
||||||
|
<span class="n">ENGINE_DEFAULT_ARGS</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span> <span class="o">|</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">|</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">|</span> <span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="c1"># Common options in the engine module</span>
|
||||||
|
<span class="s2">"engine_type"</span><span class="p">:</span> <span class="s2">"online"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"paging"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"time_range_support"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"safesearch"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="c1"># settings.yml</span>
|
||||||
|
<span class="s2">"categories"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"general"</span><span class="p">],</span>
|
||||||
|
<span class="s2">"enable_http"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"shortcut"</span><span class="p">:</span> <span class="s2">"-"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"timeout"</span><span class="p">:</span> <span class="n">settings</span><span class="p">[</span><span class="s2">"outgoing"</span><span class="p">][</span><span class="s2">"request_timeout"</span><span class="p">],</span>
|
||||||
|
<span class="s2">"display_error_messages"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"disabled"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"inactive"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"about"</span><span class="p">:</span> <span class="p">{},</span>
|
||||||
|
<span class="s2">"using_tor_proxy"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"send_accept_language_header"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"tokens"</span><span class="p">:</span> <span class="p">[],</span>
|
||||||
|
<span class="s2">"max_page"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="c1"># set automatically when an engine does not have any tab category</span>
|
||||||
|
<span class="n">DEFAULT_CATEGORY</span> <span class="o">=</span> <span class="s1">'other'</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span><span class="p">:</span> <span class="s2">"dict[str, list[Engine|types.ModuleType]]"</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'general'</span><span class="p">:</span> <span class="p">[]}</span>
|
||||||
|
|
||||||
|
<span class="n">engines</span><span class="p">:</span> <span class="s2">"dict[str, Engine | types.ModuleType]"</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="sd">"""Global registered engine instances."""</span>
|
||||||
|
|
||||||
|
<span class="n">engine_shortcuts</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="sd">"""Simple map of registered *shortcuts* to name of the engine (or ``None``).</span>
|
||||||
|
|
||||||
|
<span class="sd">::</span>
|
||||||
|
|
||||||
|
<span class="sd"> engine_shortcuts[engine.shortcut] = engine.name</span>
|
||||||
|
|
||||||
|
<span class="sd">:meta hide-value:</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">check_engine_module</span><span class="p">(</span><span class="n">module</span><span class="p">:</span> <span class="n">types</span><span class="o">.</span><span class="n">ModuleType</span><span class="p">):</span>
|
||||||
|
<span class="c1"># probe unintentional name collisions / for example name collisions caused</span>
|
||||||
|
<span class="c1"># by import statements in the engine module ..</span>
|
||||||
|
|
||||||
|
<span class="c1"># network: https://github.com/searxng/searxng/issues/762#issuecomment-1605323861</span>
|
||||||
|
<span class="n">obj</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="s1">'network'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">obj</span> <span class="ow">and</span> <span class="n">inspect</span><span class="o">.</span><span class="n">ismodule</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'type of </span><span class="si">{</span><span class="n">module</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s1">.network is a module (</span><span class="si">{</span><span class="n">obj</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s1">), expected a string'</span>
|
||||||
|
<span class="c1"># logger.error(msg)</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="load_engine">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/engines.html#searx.engines.load_engine">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">load_engine</span><span class="p">(</span><span class="n">engine_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="s2">"Engine | types.ModuleType | None"</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Load engine from ``engine_data``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param dict engine_data: Attributes from YAML ``settings:engines/<engine>``</span>
|
||||||
|
<span class="sd"> :return: initialized namespace of the ``<engine>``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> 1. create a namespace and load module of the ``<engine>``</span>
|
||||||
|
<span class="sd"> 2. update namespace with the defaults from :py:obj:`ENGINE_DEFAULT_ARGS`</span>
|
||||||
|
<span class="sd"> 3. update namespace with values from ``engine_data``</span>
|
||||||
|
|
||||||
|
<span class="sd"> If engine *is active*, return namespace of the engine, otherwise return</span>
|
||||||
|
<span class="sd"> ``None``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> This function also returns ``None`` if initialization of the namespace fails</span>
|
||||||
|
<span class="sd"> for one of the following reasons:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - engine name contains underscore</span>
|
||||||
|
<span class="sd"> - engine name is not lowercase</span>
|
||||||
|
<span class="sd"> - required attribute is not set :py:func:`is_missing_required_attributes`</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-return-statements</span>
|
||||||
|
|
||||||
|
<span class="n">engine_name</span> <span class="o">=</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'name'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'An engine does not have a "name" field'</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'_'</span> <span class="ow">in</span> <span class="n">engine_name</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Engine name contains underscore: "</span><span class="si">{}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">engine_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="n">engine_name</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s1">'Engine name is not lowercase: "</span><span class="si">{}</span><span class="s1">", converting to lowercase'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
|
||||||
|
<span class="n">engine_name</span> <span class="o">=</span> <span class="n">engine_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||||
|
<span class="n">engine_data</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_name</span>
|
||||||
|
|
||||||
|
<span class="c1"># load_module</span>
|
||||||
|
<span class="n">module_name</span> <span class="o">=</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'engine'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">module_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'The "engine" field is missing for the engine named "</span><span class="si">{}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine_name</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">engine</span> <span class="o">=</span> <span class="n">load_module</span><span class="p">(</span><span class="n">module_name</span> <span class="o">+</span> <span class="s1">'.py'</span><span class="p">,</span> <span class="n">ENGINE_DIR</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="p">(</span><span class="ne">SyntaxError</span><span class="p">,</span> <span class="ne">KeyboardInterrupt</span><span class="p">,</span> <span class="ne">SystemExit</span><span class="p">,</span> <span class="ne">SystemError</span><span class="p">,</span> <span class="ne">ImportError</span><span class="p">,</span> <span class="ne">RuntimeError</span><span class="p">):</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">'Fatal exception in engine "</span><span class="si">{}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">module_name</span><span class="p">))</span>
|
||||||
|
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">BaseException</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">'Cannot load engine "</span><span class="si">{}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">module_name</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">check_engine_module</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||||
|
<span class="n">update_engine_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">)</span>
|
||||||
|
<span class="n">update_attributes_for_tor</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># avoid cyclic imports</span>
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraitsMap</span>
|
||||||
|
|
||||||
|
<span class="n">trait_map</span> <span class="o">=</span> <span class="n">EngineTraitsMap</span><span class="o">.</span><span class="n">from_data</span><span class="p">()</span>
|
||||||
|
<span class="n">trait_map</span><span class="o">.</span><span class="n">set_traits</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_engine_active</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">is_missing_required_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">set_loggers</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">call_engine_setup</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="n">cat</span> <span class="ow">in</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'categories_as_tabs'</span><span class="p">]</span> <span class="k">for</span> <span class="n">cat</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="p">):</span>
|
||||||
|
<span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">DEFAULT_CATEGORY</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">engine</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set_loggers</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine|types.ModuleType"</span><span class="p">,</span> <span class="n">engine_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="c1"># set the logger for engine</span>
|
||||||
|
<span class="n">engine</span><span class="o">.</span><span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="n">engine_name</span><span class="p">)</span>
|
||||||
|
<span class="c1"># the engine may have load some other engines</span>
|
||||||
|
<span class="c1"># may sure the logger is initialized</span>
|
||||||
|
<span class="c1"># use sys.modules.copy() to avoid "RuntimeError: dictionary changed size during iteration"</span>
|
||||||
|
<span class="c1"># see https://github.com/python/cpython/issues/89516</span>
|
||||||
|
<span class="c1"># and https://docs.python.org/3.10/library/sys.html#sys.modules</span>
|
||||||
|
<span class="n">modules</span> <span class="o">=</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">module_name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="n">modules</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="p">(</span>
|
||||||
|
<span class="n">module_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"searx.engines"</span><span class="p">)</span>
|
||||||
|
<span class="ow">and</span> <span class="n">module_name</span> <span class="o">!=</span> <span class="s2">"searx.engines.__init__"</span>
|
||||||
|
<span class="ow">and</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="s2">"logger"</span><span class="p">)</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="n">module_engine_name</span> <span class="o">=</span> <span class="n">module_name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"."</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">module</span><span class="o">.</span><span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="n">module_engine_name</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">update_engine_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
|
||||||
|
<span class="c1"># set engine attributes from engine_data</span>
|
||||||
|
<span class="k">for</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_value</span> <span class="ow">in</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">'categories'</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">param_value</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">param_value</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="o">.</span><span class="n">strip</span><span class="p">,</span> <span class="n">param_value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)))</span>
|
||||||
|
<span class="n">engine</span><span class="o">.</span><span class="n">categories</span> <span class="o">=</span> <span class="n">param_value</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'about'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">param_name</span> <span class="o">==</span> <span class="s1">'about'</span><span class="p">:</span>
|
||||||
|
<span class="n">engine</span><span class="o">.</span><span class="n">about</span> <span class="o">=</span> <span class="p">{</span><span class="o">**</span><span class="n">engine</span><span class="o">.</span><span class="n">about</span><span class="p">,</span> <span class="o">**</span><span class="n">engine_data</span><span class="p">[</span><span class="s1">'about'</span><span class="p">]}</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="nb">setattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">param_name</span><span class="p">,</span> <span class="n">param_value</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># set default attributes</span>
|
||||||
|
<span class="k">for</span> <span class="n">arg_name</span><span class="p">,</span> <span class="n">arg_value</span> <span class="ow">in</span> <span class="n">ENGINE_DEFAULT_ARGS</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">arg_name</span><span class="p">):</span>
|
||||||
|
<span class="nb">setattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">arg_name</span><span class="p">,</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="n">arg_value</span><span class="p">))</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">update_attributes_for_tor</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'onion_url'</span><span class="p">):</span>
|
||||||
|
<span class="n">engine</span><span class="o">.</span><span class="n">search_url</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">onion_url</span> <span class="o">+</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'search_path'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">engine</span><span class="o">.</span><span class="n">timeout</span> <span class="o">+=</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'extra_proxy_timeout'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="is_missing_required_attributes">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/engines.html#searx.engines.is_missing_required_attributes">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_missing_required_attributes</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""An attribute is required when its name doesn't start with ``_`` (underline).</span>
|
||||||
|
<span class="sd"> Required attributes must not be ``None``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">missing</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="k">for</span> <span class="n">engine_attr</span> <span class="ow">in</span> <span class="nb">dir</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_attr</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="n">engine_attr</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Missing engine config attribute: "</span><span class="si">{0}</span><span class="s1">.</span><span class="si">{1}</span><span class="s1">"'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">engine_attr</span><span class="p">))</span>
|
||||||
|
<span class="n">missing</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="k">return</span> <span class="n">missing</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="using_tor_proxy">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/engines.html#searx.engines.using_tor_proxy">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Return True if the engine configuration declares to use Tor."""</span>
|
||||||
|
<span class="k">return</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'outgoing'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'using_tor_proxy'</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s1">'using_tor_proxy'</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_engine_active</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">):</span>
|
||||||
|
<span class="c1"># check if engine is inactive</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">inactive</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="c1"># exclude onion engines if not using tor</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'onions'</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">using_tor_proxy</span><span class="p">(</span><span class="n">engine</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">call_engine_setup</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">,</span> <span class="n">engine_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="n">setup_ok</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">setup_func</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">engine</span><span class="p">,</span> <span class="s2">"setup"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">setup_func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">setup_ok</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="k">elif</span> <span class="ow">not</span> <span class="nb">callable</span><span class="p">(</span><span class="n">setup_func</span><span class="p">):</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"engine's setup method isn't a callable (is of type: </span><span class="si">%s</span><span class="s2">)"</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="n">setup_func</span><span class="p">))</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">setup_ok</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">setup</span><span class="p">(</span><span class="n">engine_data</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s1">'exception : </span><span class="si">{0}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">setup_ok</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">: Engine setup was not successful, engine is set to inactive."</span><span class="p">,</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">setup_ok</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">register_engine</span><span class="p">(</span><span class="n">engine</span><span class="p">:</span> <span class="s2">"Engine | types.ModuleType"</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span> <span class="ow">in</span> <span class="n">engines</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Engine config error: ambiguous name: </span><span class="si">{0}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
|
||||||
|
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">engines</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span> <span class="ow">in</span> <span class="n">engine_shortcuts</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s1">'Engine config error: ambiguous shortcut: </span><span class="si">{0}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span><span class="p">))</span>
|
||||||
|
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">engine_shortcuts</span><span class="p">[</span><span class="n">engine</span><span class="o">.</span><span class="n">shortcut</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">name</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">category_name</span> <span class="ow">in</span> <span class="n">engine</span><span class="o">.</span><span class="n">categories</span><span class="p">:</span>
|
||||||
|
<span class="n">categories</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">category_name</span><span class="p">,</span> <span class="p">[])</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="load_engines">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/engines.html#searx.engines.load_engines">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">load_engines</span><span class="p">(</span><span class="n">engine_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""usage: ``engine_list = settings['engines']``"""</span>
|
||||||
|
<span class="n">engines</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
||||||
|
<span class="n">engine_shortcuts</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
||||||
|
<span class="n">categories</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
||||||
|
<span class="n">categories</span><span class="p">[</span><span class="s1">'general'</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">engine_data</span> <span class="ow">in</span> <span class="n">engine_list</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"inactive"</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine</span> <span class="o">=</span> <span class="n">load_engine</span><span class="p">(</span><span class="n">engine_data</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine</span><span class="p">:</span>
|
||||||
|
<span class="n">register_engine</span><span class="p">(</span><span class="n">engine</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># if an engine can't be loaded (if for example the engine is missing</span>
|
||||||
|
<span class="c1"># tor or some other requirements) its set to inactive!</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"loading engine </span><span class="si">%s</span><span class="s2"> failed: set engine to inactive!"</span><span class="p">,</span> <span class="n">engine_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"name"</span><span class="p">,</span> <span class="s2">"???"</span><span class="p">))</span>
|
||||||
|
<span class="n">engine_data</span><span class="p">[</span><span class="s2">"inactive"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="k">return</span> <span class="n">engines</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
326
_modules/searx/engines/annas_archive.html
Normal file
@@ -0,0 +1,326 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.annas_archive — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.annas_archive</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.annas_archive</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""`Anna's Archive`_ is a free non-profit online shadow library metasearch</span>
|
||||||
|
<span class="sd">engine providing access to a variety of book resources (also via IPFS), created</span>
|
||||||
|
<span class="sd">by a team of anonymous archivists (AnnaArchivist_).</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Anna's Archive: https://annas-archive.org/</span>
|
||||||
|
<span class="sd">.. _AnnaArchivist: https://annas-software.org/AnnaArchivist/annas-archive</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following additional settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`aa_content`</span>
|
||||||
|
<span class="sd">- :py:obj:`aa_ext`</span>
|
||||||
|
<span class="sd">- :py:obj:`aa_sort`</span>
|
||||||
|
|
||||||
|
<span class="sd">With this options a SearXNG maintainer is able to configure **additional**</span>
|
||||||
|
<span class="sd">engines for specific searches in Anna's Archive. For example a engine to search</span>
|
||||||
|
<span class="sd">for *newest* articles and journals (PDF) / by shortcut ``!aaa <search-term>``.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: annas articles</span>
|
||||||
|
<span class="sd"> engine: annas_archive</span>
|
||||||
|
<span class="sd"> categories = ["general", "articles"]</span>
|
||||||
|
<span class="sd"> shortcut: aaa</span>
|
||||||
|
<span class="sd"> aa_content: "magazine"</span>
|
||||||
|
<span class="sd"> aa_ext: "pdf"</span>
|
||||||
|
<span class="sd"> aa_sort: "newest"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml.etree</span><span class="w"> </span><span class="kn">import</span> <span class="n">ElementBase</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span><span class="p">,</span> <span class="n">eval_xpath_list</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">ENGINE_TRAITS</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineXPathException</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://annas-archive.org/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q115288326"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"HTML"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"files"</span><span class="p">,</span> <span class="s2">"books"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="c1"># search-url</span>
|
||||||
|
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"https://annas-archive.org"</span>
|
||||||
|
<span class="n">aa_content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Anan's search form field **Content** / possible values::</span>
|
||||||
|
|
||||||
|
<span class="sd"> book_fiction, book_unknown, book_nonfiction,</span>
|
||||||
|
<span class="sd"> book_comic, magazine, standards_document</span>
|
||||||
|
|
||||||
|
<span class="sd">To not filter use an empty string (default).</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="n">aa_sort</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Sort Anna's results, possible values::</span>
|
||||||
|
|
||||||
|
<span class="sd"> newest, oldest, largest, smallest</span>
|
||||||
|
|
||||||
|
<span class="sd">To sort by *most relevant* use an empty string (default)."""</span>
|
||||||
|
|
||||||
|
<span class="n">aa_ext</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Filter Anna's results by a file ending. Common filters for example are</span>
|
||||||
|
<span class="sd">``pdf`` and ``epub``.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> Anna's Archive is a beta release: Filter results by file extension does not</span>
|
||||||
|
<span class="sd"> really work on Anna's Archive.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/annas_archive.html#searx.engines.annas_archive.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Check of engine's settings."""</span>
|
||||||
|
<span class="n">traits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">"annas archive"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">aa_content</span> <span class="ow">and</span> <span class="n">aa_content</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"content"</span><span class="p">]:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"invalid setting content: </span><span class="si">{</span><span class="n">aa_content</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">aa_sort</span> <span class="ow">and</span> <span class="n">aa_sort</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"sort"</span><span class="p">]:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"invalid setting sort: </span><span class="si">{</span><span class="n">aa_sort</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">aa_ext</span> <span class="ow">and</span> <span class="n">aa_ext</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ext"</span><span class="p">]:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"invalid setting ext: </span><span class="si">{</span><span class="n">aa_ext</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"searxng_locale"</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"lang"</span><span class="p">:</span> <span class="n">lang</span><span class="p">,</span>
|
||||||
|
<span class="s2">"content"</span><span class="p">:</span> <span class="n">aa_content</span><span class="p">,</span>
|
||||||
|
<span class="s2">"ext"</span><span class="p">:</span> <span class="n">aa_ext</span><span class="p">,</span>
|
||||||
|
<span class="s2">"sort"</span><span class="p">:</span> <span class="n">aa_sort</span><span class="p">,</span>
|
||||||
|
<span class="s2">"q"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"page"</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="c1"># filter out None and empty values</span>
|
||||||
|
<span class="n">filtered_args</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">args</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">v</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">filtered_args</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># The rendering of the WEB page is strange; positions of Anna's result page</span>
|
||||||
|
<span class="c1"># are enclosed in SGML comments. These comments are *uncommented* by some</span>
|
||||||
|
<span class="c1"># JS code, see query of class ".js-scroll-hidden" in Anna's HTML template:</span>
|
||||||
|
<span class="c1"># https://annas-software.org/AnnaArchivist/annas-archive/-/blob/main/allthethings/templates/macros/md5_list.html</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//main//div[contains(@class, 'js-aarecord-list-outer')]/div"</span><span class="p">):</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">kwargs</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">_get_result</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">SearxEngineXPathException</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_get_result</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">ElementBase</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
|
||||||
|
<span class="k">return</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"url"</span><span class="p">:</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">"./a/@href"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
|
||||||
|
<span class="s2">"title"</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">"./div//a[starts-with(@href, '/md5')]"</span><span class="p">)),</span>
|
||||||
|
<span class="s2">"authors"</span><span class="p">:</span> <span class="p">[</span><span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">".//a[starts-with(@href, '/search')]"</span><span class="p">,</span> <span class="mi">0</span><span class="p">))],</span>
|
||||||
|
<span class="s2">"publisher"</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">".//a[starts-with(@href, '/search')]"</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="s2">"content"</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">".//div[contains(@class, 'relative')]"</span><span class="p">)),</span>
|
||||||
|
<span class="s2">"thumbnail"</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">".//img/@src"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/annas_archive.html#searx.engines.annas_archive.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages and other search arguments from Anna's search form."""</span>
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"content"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ext"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"sort"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">base_url</span> <span class="o">+</span> <span class="s2">"/search"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"Response from Anna's search page is not OK."</span><span class="p">)</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># supported language codes</span>
|
||||||
|
|
||||||
|
<span class="n">lang_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//form//input[@name='lang']"</span><span class="p">):</span>
|
||||||
|
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_lang</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">""</span><span class="p">,</span> <span class="s2">"_empty"</span><span class="p">,</span> <span class="s2">"nl-BE"</span><span class="p">,</span> <span class="s2">"und"</span><span class="p">)</span> <span class="ow">or</span> <span class="n">eng_lang</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"anti__"</span><span class="p">):</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s2">"-"</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="c1"># silently ignore unknown languages</span>
|
||||||
|
<span class="c1"># print("ERROR: %s -> %s is unknown by babel" % (x.get("data-name"), eng_lang))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_lang</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_lang</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//form//input[@name='content']"</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"anti__"</span><span class="p">):</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"content"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//form//input[@name='ext']"</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"anti__"</span><span class="p">):</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ext"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//form//select[@name='sort']//option"</span><span class="p">):</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"sort"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># for better diff; sort the persistence of these traits</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"content"</span><span class="p">]</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ext"</span><span class="p">]</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"sort"</span><span class="p">]</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
257
_modules/searx/engines/archlinux.html
Normal file
@@ -0,0 +1,257 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.archlinux — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.archlinux</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.archlinux</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Arch Linux Wiki</span>
|
||||||
|
<span class="sd">~~~~~~~~~~~~~~~</span>
|
||||||
|
|
||||||
|
<span class="sd">This implementation does not use a official API: Mediawiki provides API, but</span>
|
||||||
|
<span class="sd">Arch Wiki blocks access to it.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urljoin</span><span class="p">,</span> <span class="n">urlparse</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span><span class="p">,</span> <span class="n">searxng_useragent</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://wiki.archlinux.org/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q101445877'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'it'</span><span class="p">,</span> <span class="s1">'software wikis'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">main_wiki</span> <span class="o">=</span> <span class="s1">'wiki.archlinux.org'</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">netloc</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">main_wiki</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="s1">'Special:Search'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://'</span> <span class="o">+</span> <span class="n">netloc</span> <span class="o">+</span> <span class="s1">'/index.php?'</span>
|
||||||
|
<span class="n">offset</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">20</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">netloc</span> <span class="o">==</span> <span class="n">main_wiki</span><span class="p">:</span>
|
||||||
|
<span class="n">eng_lang</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="s1">'English'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">query</span> <span class="o">+=</span> <span class="s1">' ('</span> <span class="o">+</span> <span class="n">eng_lang</span> <span class="o">+</span> <span class="s1">')'</span>
|
||||||
|
<span class="c1"># wiki.archlinux.org is protected by anubis</span>
|
||||||
|
<span class="c1"># - https://github.com/searxng/searxng/issues/4646#issuecomment-2817848019</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'User-Agent'</span><span class="p">]</span> <span class="o">=</span> <span class="n">searxng_useragent</span><span class="p">()</span>
|
||||||
|
<span class="k">elif</span> <span class="n">netloc</span> <span class="o">==</span> <span class="s1">'wiki.archlinuxcn.org'</span><span class="p">:</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://'</span> <span class="o">+</span> <span class="n">netloc</span> <span class="o">+</span> <span class="s1">'/wzh/index.php?'</span>
|
||||||
|
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'search'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'limit'</span><span class="p">:</span> <span class="mi">20</span><span class="p">,</span>
|
||||||
|
<span class="s1">'offset'</span><span class="p">:</span> <span class="n">offset</span><span class="p">,</span>
|
||||||
|
<span class="s1">'profile'</span><span class="p">:</span> <span class="s1">'default'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="c1"># get the base URL for the language in which request was made</span>
|
||||||
|
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">netloc</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">main_wiki</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://'</span> <span class="o">+</span> <span class="n">netloc</span> <span class="o">+</span> <span class="s1">'/index.php?'</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//ul[@class="mw-search-results"]/li'</span><span class="p">):</span>
|
||||||
|
<span class="n">link</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[@class="mw-search-result-heading"]/a'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//div[@class="searchresult"]'</span><span class="p">))</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">urljoin</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">link</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'href'</span><span class="p">)),</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">link</span><span class="p">),</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/archlinux.html#searx.engines.archlinux.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages from Archlinux-Wiki. The location of the Wiki address of a</span>
|
||||||
|
<span class="sd"> language is mapped in a :py:obj:`custom field</span>
|
||||||
|
<span class="sd"> <searx.enginelib.traits.EngineTraits.custom>` (``wiki_netloc``). Depending</span>
|
||||||
|
<span class="sd"> on the location, the ``title`` argument in the request is translated.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> "custom": {</span>
|
||||||
|
<span class="sd"> "wiki_netloc": {</span>
|
||||||
|
<span class="sd"> "de": "wiki.archlinux.de",</span>
|
||||||
|
<span class="sd"> # ...</span>
|
||||||
|
<span class="sd"> "zh": "wiki.archlinuxcn.org"</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
<span class="sd"> "title": {</span>
|
||||||
|
<span class="sd"> "de": "Spezial:Suche",</span>
|
||||||
|
<span class="sd"> # ...</span>
|
||||||
|
<span class="sd"> "zh": "Special:\u641c\u7d22"</span>
|
||||||
|
<span class="sd"> },</span>
|
||||||
|
<span class="sd"> },</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="n">title_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'de'</span><span class="p">:</span> <span class="s1">'Spezial:Suche'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'fa'</span><span class="p">:</span> <span class="s1">'ویژه:جستجو'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ja'</span><span class="p">:</span> <span class="s1">'特別:検索'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'zh'</span><span class="p">:</span> <span class="s1">'Special:搜索'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">'https://wiki.archlinux.org/'</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from wiki.archlinux.org is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//a[@class='interlanguage-link-target']"</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'lang'</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">))</span>
|
||||||
|
<span class="c1"># zh_Hans --> zh</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">sxng_tag</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">netloc</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'href'</span><span class="p">))</span><span class="o">.</span><span class="n">netloc</span>
|
||||||
|
<span class="k">if</span> <span class="n">netloc</span> <span class="o">!=</span> <span class="s1">'wiki.archlinux.org'</span><span class="p">:</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">title_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">title</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: title tag from </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) is unknown"</span> <span class="o">%</span> <span class="p">(</span><span class="n">netloc</span><span class="p">,</span> <span class="n">sxng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">][</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">netloc</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'title'</span><span class="p">][</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">title</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="s2">".//span"</span><span class="p">))</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">'en'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'English'</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
274
_modules/searx/engines/astrophysics_data_system.html
Normal file
@@ -0,0 +1,274 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.astrophysics_data_system — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.astrophysics_data_system</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.astrophysics_data_system</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""The Astrophysics Data System (ADS_) is a digital library portal for</span>
|
||||||
|
<span class="sd">researchers in astronomy and physics, operated by the Smithsonian Astrophysical</span>
|
||||||
|
<span class="sd">Observatory (SAO) under a NASA grant. The ADS_ is a solr instance, but not with</span>
|
||||||
|
<span class="sd">the standard API paths.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> The ADS_ engine requires an :py:obj:`API key <api_key>`.</span>
|
||||||
|
|
||||||
|
<span class="sd">This engine uses the `search/query`_ API endpoint. Since the user's search term</span>
|
||||||
|
<span class="sd">is passed through, the `search syntax`_ of ADS can be used (at least to some</span>
|
||||||
|
<span class="sd">extent).</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _ADS: https://ui.adsabs.harvard.edu</span>
|
||||||
|
<span class="sd">.. _search/query: https://ui.adsabs.harvard.edu/help/api/api-docs.html#get-/search/query</span>
|
||||||
|
<span class="sd">.. _search syntax: https://ui.adsabs.harvard.edu/help/search/search-syntax</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following additional settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`api_key`</span>
|
||||||
|
<span class="sd">- :py:obj:`ads_sort`</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: astrophysics data system</span>
|
||||||
|
<span class="sd"> api_key: "..."</span>
|
||||||
|
<span class="sd"> inactive: false</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineAPIException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://ui.adsabs.harvard.edu/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q752099"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"https://ui.adsabs.harvard.edu/help/api/api-docs.html"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"science"</span><span class="p">,</span> <span class="s2">"scientific publications"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://api.adsabs.harvard.edu/v1/search/query"</span>
|
||||||
|
|
||||||
|
<span class="n">api_key</span> <span class="o">=</span> <span class="s2">"unset"</span>
|
||||||
|
<span class="sd">"""Get an API token as described in https://ui.adsabs.harvard.edu/help/api"""</span>
|
||||||
|
|
||||||
|
<span class="n">ads_field_list</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="s2">"abstract"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"author"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"bibcode"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"comment"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"date"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"doi"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"isbn"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"issn"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"keyword"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"page"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"page_count"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"page_range"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"pub"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"pubdate"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"pubnote"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"read_count"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"title"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"volume"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"year"</span><span class="p">,</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="sd">"""Set of fields to return in the response from ADS."""</span>
|
||||||
|
|
||||||
|
<span class="n">ads_rows</span> <span class="o">=</span> <span class="mi">10</span>
|
||||||
|
<span class="sd">"""How many records to return for the ADS request."""</span>
|
||||||
|
|
||||||
|
<span class="n">ads_sort</span> <span class="o">=</span> <span class="s2">"read_count desc"</span>
|
||||||
|
<span class="sd">"""The format is 'field' + 'direction' where direction is one of 'asc' or 'desc'</span>
|
||||||
|
<span class="sd">and field is any of the valid indexes."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/astrophysics_data_system.html#searx.engines.astrophysics_data_system.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the ADS_ engine, checks whether the :py:obj:`api_key`</span>
|
||||||
|
<span class="sd"> is set, otherwise the engine is inactive.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"api_key"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">key</span> <span class="ow">and</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"unset"</span><span class="p">,</span> <span class="s2">"unknown"</span><span class="p">,</span> <span class="s2">"..."</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Astrophysics Data System (ADS) API key is not set or invalid."</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">args</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"q"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"fl"</span><span class="p">:</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">ads_field_list</span><span class="p">),</span>
|
||||||
|
<span class="s2">"rows"</span><span class="p">:</span> <span class="n">ads_rows</span><span class="p">,</span>
|
||||||
|
<span class="s2">"start"</span><span class="p">:</span> <span class="n">ads_rows</span> <span class="o">*</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="k">if</span> <span class="n">ads_sort</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s2">"sort"</span><span class="p">]</span> <span class="o">=</span> <span class="n">ads_sort</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">][</span><span class="s2">"Authorization"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Bearer </span><span class="si">{</span><span class="n">api_key</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">json_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s2">"error"</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="n">json_data</span><span class="p">[</span><span class="s2">"error"</span><span class="p">][</span><span class="s2">"msg"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_str</span><span class="p">(</span><span class="n">k</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">doc</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="s2">""</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_list</span><span class="p">(</span><span class="n">k</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="k">return</span> <span class="n">doc</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">doc</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">"response"</span><span class="p">][</span><span class="s2">"docs"</span><span class="p">]:</span>
|
||||||
|
<span class="n">authors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">doc</span><span class="p">[</span><span class="s2">"author"</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">authors</span><span class="p">)</span> <span class="o">></span> <span class="mi">15</span><span class="p">:</span>
|
||||||
|
<span class="c1"># There are articles with hundreds of authors</span>
|
||||||
|
<span class="n">authors</span> <span class="o">=</span> <span class="n">authors</span><span class="p">[:</span><span class="mi">15</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="s2">"et al."</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">paper</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="sa">f</span><span class="s2">"https://ui.adsabs.harvard.edu/abs/</span><span class="si">{</span><span class="n">doc</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'bibcode'</span><span class="p">)</span><span class="si">}</span><span class="s2">/"</span><span class="p">,</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">_list</span><span class="p">(</span><span class="s2">"title"</span><span class="p">)[</span><span class="mi">0</span><span class="p">]),</span>
|
||||||
|
<span class="n">authors</span><span class="o">=</span><span class="n">authors</span><span class="p">,</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="s2">"abstract"</span><span class="p">)),</span>
|
||||||
|
<span class="n">doi</span><span class="o">=</span><span class="n">_list</span><span class="p">(</span><span class="s2">"doi"</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
|
||||||
|
<span class="n">issn</span><span class="o">=</span><span class="n">_list</span><span class="p">(</span><span class="s2">"issn"</span><span class="p">),</span>
|
||||||
|
<span class="n">isbn</span><span class="o">=</span><span class="n">_list</span><span class="p">(</span><span class="s2">"isbn"</span><span class="p">),</span>
|
||||||
|
<span class="n">tags</span><span class="o">=</span><span class="n">_list</span><span class="p">(</span><span class="s2">"keyword"</span><span class="p">),</span>
|
||||||
|
<span class="n">pages</span><span class="o">=</span><span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_list</span><span class="p">(</span><span class="s2">"page"</span><span class="p">)),</span>
|
||||||
|
<span class="n">publisher</span><span class="o">=</span><span class="n">_str</span><span class="p">(</span><span class="s2">"pub"</span><span class="p">)</span> <span class="o">+</span> <span class="s2">" "</span> <span class="o">+</span> <span class="n">_str</span><span class="p">(</span><span class="s2">"year"</span><span class="p">),</span>
|
||||||
|
<span class="n">publishedDate</span><span class="o">=</span><span class="n">datetime</span><span class="o">.</span><span class="n">fromisoformat</span><span class="p">(</span><span class="n">_str</span><span class="p">(</span><span class="s2">"date"</span><span class="p">)),</span>
|
||||||
|
<span class="n">volume</span><span class="o">=</span><span class="n">_str</span><span class="p">(</span><span class="s2">"volume"</span><span class="p">),</span>
|
||||||
|
<span class="n">views</span><span class="o">=</span><span class="n">_str</span><span class="p">(</span><span class="s2">"read_count"</span><span class="p">),</span>
|
||||||
|
<span class="n">comments</span><span class="o">=</span><span class="s2">" / "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_list</span><span class="p">(</span><span class="s2">"pubnote"</span><span class="p">)),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">paper</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
304
_modules/searx/engines/azure.html
Normal file
@@ -0,0 +1,304 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.azure — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.azure</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.azure</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Engine for Azure resources. This engine mimics the standard search bar in Azure</span>
|
||||||
|
<span class="sd">Portal (for resources and resource groups).</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">You must `register an application in Microsoft Entra ID`_ and assign it the</span>
|
||||||
|
<span class="sd">'Reader' role in your subscription.</span>
|
||||||
|
|
||||||
|
<span class="sd">To use this engine, add an entry similar to the following to your engine list in</span>
|
||||||
|
<span class="sd">``settings.yml``:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: azure</span>
|
||||||
|
<span class="sd"> engine: azure</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> azure_tenant_id: "your_tenant_id"</span>
|
||||||
|
<span class="sd"> azure_client_id: "your_client_id"</span>
|
||||||
|
<span class="sd"> azure_client_secret: "your_client_secret"</span>
|
||||||
|
<span class="sd"> azure_token_expiration_seconds: 5000</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _register an application in Microsoft Entra ID:</span>
|
||||||
|
<span class="sd"> https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">post</span> <span class="k">as</span> <span class="n">http_post</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
<span class="n">engine_type</span> <span class="o">=</span> <span class="s2">"online"</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"it"</span><span class="p">,</span> <span class="s2">"cloud"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Default values, should be overridden in settings.yml</span>
|
||||||
|
<span class="n">azure_tenant_id</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">azure_client_id</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">azure_client_secret</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">azure_token_expiration_seconds</span> <span class="o">=</span> <span class="mi">5000</span>
|
||||||
|
<span class="sd">"""Time for which an auth token is valid (sec.)"""</span>
|
||||||
|
<span class="n">azure_batch_endpoint</span> <span class="o">=</span> <span class="s2">"https://management.azure.com/batch?api-version=2020-06-01"</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://www.portal.azure.com"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q725967"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"https://learn.microsoft.com/en-us/</span><span class="se">\</span>
|
||||||
|
<span class="s2"> rest/api/azure-resourcegraph/?view=rest-azureresourcegraph-resourcegraph-2024-04-01"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"language"</span><span class="p">:</span> <span class="s2">"en"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">CACHE</span><span class="p">:</span> <span class="n">EngineCache</span>
|
||||||
|
<span class="sd">"""Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
|
||||||
|
<span class="sd">seconds."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/azure.html#searx.engines.azure.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - Instantiate a cache for this engine (:py:obj:`CACHE`).</span>
|
||||||
|
<span class="sd"> - Checks whether the tenant_id, client_id and client_secret are set,</span>
|
||||||
|
<span class="sd"> otherwise the engine is inactive.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">[</span><span class="s2">"name"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="n">missing_opts</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">opt</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"azure_tenant_id"</span><span class="p">,</span> <span class="s2">"azure_client_id"</span><span class="p">,</span> <span class="s2">"azure_client_secret"</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">opt</span><span class="p">,</span> <span class="s2">""</span><span class="p">):</span>
|
||||||
|
<span class="n">missing_opts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">opt</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">missing_opts</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"missing values for options: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">missing_opts</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="authenticate">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/azure.html#searx.engines.azure.authenticate">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">authenticate</span><span class="p">(</span><span class="n">t_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">c_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">c_secret</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Authenticates to Azure using Oauth2 Client Credentials Flow and returns</span>
|
||||||
|
<span class="sd"> an access token."""</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://login.microsoftonline.com/</span><span class="si">{</span><span class="n">t_id</span><span class="si">}</span><span class="s2">/oauth2/v2.0/token"</span>
|
||||||
|
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"client_id"</span><span class="p">:</span> <span class="n">c_id</span><span class="p">,</span>
|
||||||
|
<span class="s2">"client_secret"</span><span class="p">:</span> <span class="n">c_secret</span><span class="p">,</span>
|
||||||
|
<span class="s2">"grant_type"</span><span class="p">:</span> <span class="s2">"client_credentials"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"scope"</span><span class="p">:</span> <span class="s2">"https://management.azure.com/.default"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span> <span class="o">=</span> <span class="n">http_post</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">body</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Azure authentication failed (status </span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">status_code</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="s2">"access_token"</span><span class="p">]</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_auth_token</span><span class="p">(</span><span class="n">t_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">c_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">c_secret</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="n">key</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"azure_tenant_id: </span><span class="si">{</span><span class="n">t_id</span><span class="si">:}</span><span class="s2">, azure_client_id: </span><span class="si">{</span><span class="n">c_id</span><span class="si">}</span><span class="s2">, azure_client_secret: </span><span class="si">{</span><span class="n">c_secret</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">token</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">token</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">token</span>
|
||||||
|
<span class="n">token</span> <span class="o">=</span> <span class="n">authenticate</span><span class="p">(</span><span class="n">t_id</span><span class="p">,</span> <span class="n">c_id</span><span class="p">,</span> <span class="n">c_secret</span><span class="p">)</span>
|
||||||
|
<span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">token</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="n">azure_token_expiration_seconds</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">token</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">token</span> <span class="o">=</span> <span class="n">get_auth_token</span><span class="p">(</span><span class="n">azure_tenant_id</span><span class="p">,</span> <span class="n">azure_client_id</span><span class="p">,</span> <span class="n">azure_client_secret</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="n">azure_batch_endpoint</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"method"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"POST"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">][</span><span class="s2">"Authorization"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Bearer </span><span class="si">{</span><span class="n">token</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">][</span><span class="s2">"Content-Type"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"application/json"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"json"</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"requests"</span><span class="p">:</span> <span class="p">[</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s2">"url"</span><span class="p">:</span> <span class="s2">"/providers/Microsoft.ResourceGraph/resources?api-version=2024-04-01"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"httpMethod"</span><span class="p">:</span> <span class="s2">"POST"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"name"</span><span class="p">:</span> <span class="s2">"resourceGroups"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"requestHeaderDetails"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"commandName"</span><span class="p">:</span> <span class="s2">"Microsoft.ResourceGraph"</span><span class="p">},</span>
|
||||||
|
<span class="s2">"content"</span><span class="p">:</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"query"</span><span class="p">:</span> <span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"ResourceContainers"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" | where (name contains ('</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">'))"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" | where (type =~ ('Microsoft.Resources/subscriptions/resourcegroups'))"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" | project id,name,type,kind,subscriptionId,resourceGroup"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" | extend matchscore = name startswith '</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">'"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" | extend normalizedName = tolower(tostring(name))"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" | sort by matchscore desc, normalizedName asc"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" | take 30"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">},</span>
|
||||||
|
<span class="p">},</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s2">"url"</span><span class="p">:</span> <span class="s2">"/providers/Microsoft.ResourceGraph/resources?api-version=2024-04-01"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"httpMethod"</span><span class="p">:</span> <span class="s2">"POST"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"name"</span><span class="p">:</span> <span class="s2">"resources"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"requestHeaderDetails"</span><span class="p">:</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"commandName"</span><span class="p">:</span> <span class="s2">"Microsoft.ResourceGraph"</span><span class="p">,</span>
|
||||||
|
<span class="p">},</span>
|
||||||
|
<span class="s2">"content"</span><span class="p">:</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"query"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"Resources | where name contains '</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">' | take 30"</span><span class="p">,</span>
|
||||||
|
<span class="p">},</span>
|
||||||
|
<span class="p">},</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">"responses"</span><span class="p">]:</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"resourceGroups"</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">data</span> <span class="ow">in</span> <span class="n">result</span><span class="p">[</span><span class="s2">"content"</span><span class="p">][</span><span class="s2">"data"</span><span class="p">]:</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"https://portal.azure.com/#@/resource"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"/subscriptions/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'subscriptionId'</span><span class="p">]</span><span class="si">}</span><span class="s2">/resourceGroups/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span><span class="si">}</span><span class="s2">/overview"</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">data</span><span class="p">[</span><span class="s2">"name"</span><span class="p">],</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="sa">f</span><span class="s2">"Resource Group in Subscription: </span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'subscriptionId'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">elif</span> <span class="n">result</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"resources"</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">data</span> <span class="ow">in</span> <span class="n">result</span><span class="p">[</span><span class="s2">"content"</span><span class="p">][</span><span class="s2">"data"</span><span class="p">]:</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"https://portal.azure.com/#@/resource"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"/subscriptions/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'subscriptionId'</span><span class="p">]</span><span class="si">}</span><span class="s2">/resourceGroups/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'resourceGroup'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"/providers/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span><span class="si">}</span><span class="s2">/overview"</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">data</span><span class="p">[</span><span class="s2">"name"</span><span class="p">],</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"Resource of type </span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span><span class="si">}</span><span class="s2"> in Subscription:"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'subscriptionId'</span><span class="p">]</span><span class="si">}</span><span class="s2">, Resource Group: </span><span class="si">{</span><span class="n">data</span><span class="p">[</span><span class="s1">'resourceGroup'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
393
_modules/searx/engines/bing.html
Normal file
@@ -0,0 +1,393 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.bing — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.bing</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.bing</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This is the implementation of the Bing-WEB engine. Some of this</span>
|
||||||
|
<span class="sd">implementations are shared by other engines:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :ref:`bing images engine`</span>
|
||||||
|
<span class="sd">- :ref:`bing news engine`</span>
|
||||||
|
<span class="sd">- :ref:`bing videos engine`</span>
|
||||||
|
|
||||||
|
<span class="sd">On the `preference page`_ Bing offers a lot of languages an regions (see section</span>
|
||||||
|
<span class="sd">LANGUAGE and COUNTRY/REGION). The Language is the language of the UI, we need</span>
|
||||||
|
<span class="sd">in SearXNG to get the translations of data such as *"published last week"*.</span>
|
||||||
|
|
||||||
|
<span class="sd">There is a description of the official search-APIs_, unfortunately this is not</span>
|
||||||
|
<span class="sd">the API we can use or that bing itself would use. You can look up some things</span>
|
||||||
|
<span class="sd">in the API to get a better picture of bing, but the value specifications like</span>
|
||||||
|
<span class="sd">the market codes are usually outdated or at least no longer used by bing itself.</span>
|
||||||
|
|
||||||
|
<span class="sd">The market codes have been harmonized and are identical for web, video and</span>
|
||||||
|
<span class="sd">images. The news area has also been harmonized with the other categories. Only</span>
|
||||||
|
<span class="sd">political adjustments still seem to be made -- for example, there is no news</span>
|
||||||
|
<span class="sd">category for the Chinese market.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _preference page: https://www.bing.com/account/general</span>
|
||||||
|
<span class="sd">.. _search-APIs: https://learn.microsoft.com/en-us/bing/search-apis/</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches, invalid-name</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse_qs</span><span class="p">,</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urlparse</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.languages</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span><span class="p">,</span> <span class="n">region_tag</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineAPIException</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.bing.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q182496'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://www.microsoft.com/en-us/bing/apis/bing-web-search-api'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">200</span>
|
||||||
|
<span class="sd">"""200 pages maximum (``&first=1991``)"""</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="sd">"""Bing results are always SFW. To get NSFW links from bing some age</span>
|
||||||
|
<span class="sd">verification by a cookie is needed / thats not possible in SearXNG.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://www.bing.com/search'</span>
|
||||||
|
<span class="sd">"""Bing (Web) search URL"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_page_offset</span><span class="p">(</span><span class="n">pageno</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">pageno</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span> <span class="o">+</span> <span class="mi">1</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">):</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'_EDGE_CD'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'m=</span><span class="si">{</span><span class="n">engine_region</span><span class="si">}</span><span class="s1">&u=</span><span class="si">{</span><span class="n">engine_language</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'_EDGE_S'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'mkt=</span><span class="si">{</span><span class="n">engine_region</span><span class="si">}</span><span class="s1">&ui=</span><span class="si">{</span><span class="n">engine_language</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"bing cookies: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">])</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Assemble a Bing-Web request."""</span>
|
||||||
|
|
||||||
|
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="s1">'en'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">page</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'pageno'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="c1"># if arg 'pq' is missed, sometimes on page 4 we get results from page 1,</span>
|
||||||
|
<span class="c1"># don't ask why it is only sometimes / its M$ and they have never been</span>
|
||||||
|
<span class="c1"># deterministic ;)</span>
|
||||||
|
<span class="s1">'pq'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># To get correct page, arg first and this arg FORM is needed, the value PERE</span>
|
||||||
|
<span class="c1"># is on page 2, on page 3 its PERE1 and on page 4 its PERE2 .. and so forth.</span>
|
||||||
|
<span class="c1"># The 'first' arg should never send on page 1.</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">page</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'first'</span><span class="p">]</span> <span class="o">=</span> <span class="n">_page_offset</span><span class="p">(</span><span class="n">page</span><span class="p">)</span> <span class="c1"># see also arg FORM</span>
|
||||||
|
<span class="k">if</span> <span class="n">page</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'FORM'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'PERE'</span>
|
||||||
|
<span class="k">elif</span> <span class="n">page</span> <span class="o">></span> <span class="mi">2</span><span class="p">:</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'FORM'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'PERE</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">page</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s1">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'time_range'</span><span class="p">):</span>
|
||||||
|
<span class="n">unix_day</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">/</span> <span class="mi">86400</span><span class="p">)</span>
|
||||||
|
<span class="n">time_ranges</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'day'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span> <span class="s1">'week'</span><span class="p">:</span> <span class="s1">'2'</span><span class="p">,</span> <span class="s1">'month'</span><span class="p">:</span> <span class="s1">'3'</span><span class="p">,</span> <span class="s1">'year'</span><span class="p">:</span> <span class="sa">f</span><span class="s1">'5_</span><span class="si">{</span><span class="n">unix_day</span><span class="o">-</span><span class="mi">365</span><span class="si">}</span><span class="s1">_</span><span class="si">{</span><span class="n">unix_day</span><span class="si">}</span><span class="s1">'</span><span class="p">}</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">+=</span> <span class="sa">f</span><span class="s1">'&filters=ex1:"ez</span><span class="si">{</span><span class="n">time_ranges</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s2">"time_range"</span><span class="p">]]</span><span class="si">}</span><span class="s1">"'</span>
|
||||||
|
|
||||||
|
<span class="c1"># in some regions where geoblocking is employed (e.g. China),</span>
|
||||||
|
<span class="c1"># www.bing.com redirects to the regional version of Bing</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'allow_redirects'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-locals</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">result_len</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse results again if nothing is found yet</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//ol[@id="b_results"]/li[contains(@class, "b_algo")]'</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">link</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//h2/a'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">link</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">link</span><span class="o">.</span><span class="n">attrib</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'href'</span><span class="p">)</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">link</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//p'</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">content</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Make sure that the element is free of:</span>
|
||||||
|
<span class="c1"># <span class="algoSlug_icon" # data-priority="2">Web</span></span>
|
||||||
|
<span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">p</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//span[@class="algoSlug_icon"]'</span><span class="p">):</span>
|
||||||
|
<span class="n">e</span><span class="o">.</span><span class="n">getparent</span><span class="p">()</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># get the real URL</span>
|
||||||
|
<span class="k">if</span> <span class="n">url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'https://www.bing.com/ck/a?'</span><span class="p">):</span>
|
||||||
|
<span class="c1"># get the first value of u parameter</span>
|
||||||
|
<span class="n">url_query</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">query</span>
|
||||||
|
<span class="n">parsed_url_query</span> <span class="o">=</span> <span class="n">parse_qs</span><span class="p">(</span><span class="n">url_query</span><span class="p">)</span>
|
||||||
|
<span class="n">param_u</span> <span class="o">=</span> <span class="n">parsed_url_query</span><span class="p">[</span><span class="s2">"u"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="c1"># remove "a1" in front</span>
|
||||||
|
<span class="n">encoded_url</span> <span class="o">=</span> <span class="n">param_u</span><span class="p">[</span><span class="mi">2</span><span class="p">:]</span>
|
||||||
|
<span class="c1"># add padding</span>
|
||||||
|
<span class="n">encoded_url</span> <span class="o">=</span> <span class="n">encoded_url</span> <span class="o">+</span> <span class="s1">'='</span> <span class="o">*</span> <span class="p">(</span><span class="o">-</span><span class="nb">len</span><span class="p">(</span><span class="n">encoded_url</span><span class="p">)</span> <span class="o">%</span> <span class="mi">4</span><span class="p">)</span>
|
||||||
|
<span class="c1"># decode base64 encoded URL</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">urlsafe_b64decode</span><span class="p">(</span><span class="n">encoded_url</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># append result</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="c1"># get number_of_results</span>
|
||||||
|
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
|
||||||
|
<span class="n">result_len_container</span> <span class="o">=</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//span[@class="sb_count"]//text()'</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="s2">"-"</span> <span class="ow">in</span> <span class="n">result_len_container</span><span class="p">:</span>
|
||||||
|
<span class="n">start_str</span><span class="p">,</span> <span class="n">result_len_container</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'-\d+'</span><span class="p">,</span> <span class="n">result_len_container</span><span class="p">)</span>
|
||||||
|
<span class="n">start</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">start_str</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">start</span> <span class="o">=</span> <span class="mi">1</span>
|
||||||
|
|
||||||
|
<span class="n">result_len_container</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s1">'[^0-9]'</span><span class="p">,</span> <span class="s1">''</span><span class="p">,</span> <span class="n">result_len_container</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">result_len_container</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">result_len</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">result_len_container</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">expected_start</span> <span class="o">=</span> <span class="n">_page_offset</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"pageno"</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">expected_start</span> <span class="o">!=</span> <span class="n">start</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">expected_start</span> <span class="o">></span> <span class="n">result_len</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Avoid reading more results than available.</span>
|
||||||
|
<span class="c1"># For example, if there is 100 results from some search and we try to get results from 120 to 130,</span>
|
||||||
|
<span class="c1"># Bing will send back the results from 0 to 10 and no error.</span>
|
||||||
|
<span class="c1"># If we compare results count with the first parameter of the request we can avoid this "invalid"</span>
|
||||||
|
<span class="c1"># results.</span>
|
||||||
|
<span class="k">return</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Sometimes Bing will send back the first result page instead of the requested page as a rate limiting</span>
|
||||||
|
<span class="c1"># measure.</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Expected results to start at </span><span class="si">{</span><span class="n">expected_start</span><span class="si">}</span><span class="s2">, but got results starting at </span><span class="si">{</span><span class="n">start</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'number_of_results'</span><span class="p">:</span> <span class="n">result_len</span><span class="p">})</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages and regions from Bing-Web."""</span>
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">gen_useragent</span>
|
||||||
|
|
||||||
|
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="n">gen_useragent</span><span class="p">(),</span>
|
||||||
|
<span class="s2">"Accept"</span><span class="p">:</span> <span class="s2">"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Accept-Language"</span><span class="p">:</span> <span class="s2">"en-US;q=0.5,en;q=0.3"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"DNT"</span><span class="p">:</span> <span class="s2">"1"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Connection"</span><span class="p">:</span> <span class="s2">"keep-alive"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Upgrade-Insecure-Requests"</span><span class="p">:</span> <span class="s2">"1"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Sec-GPC"</span><span class="p">:</span> <span class="s2">"1"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Cache-Control"</span><span class="p">:</span> <span class="s2">"max-age=0"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s2">"https://www.bing.com/account/general"</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from bing is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="c1"># languages</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">'zh'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'zh-hans'</span>
|
||||||
|
|
||||||
|
<span class="n">map_lang</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'prs'</span><span class="p">:</span> <span class="s1">'fa-AF'</span><span class="p">,</span> <span class="s1">'en'</span><span class="p">:</span> <span class="s1">'en-us'</span><span class="p">}</span>
|
||||||
|
<span class="n">bing_ui_lang_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="c1"># HINT: this list probably needs to be supplemented</span>
|
||||||
|
<span class="s1">'en'</span><span class="p">:</span> <span class="s1">'us'</span><span class="p">,</span> <span class="c1"># en --> en-us</span>
|
||||||
|
<span class="s1">'da'</span><span class="p">:</span> <span class="s1">'dk'</span><span class="p">,</span> <span class="c1"># da --> da-dk</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">href</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[@id="language-section-content"]//div[@class="languageItem"]/a/@href'</span><span class="p">):</span>
|
||||||
|
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">parse_qs</span><span class="p">(</span><span class="n">urlparse</span><span class="p">(</span><span class="n">href</span><span class="p">)</span><span class="o">.</span><span class="n">query</span><span class="p">)[</span><span class="s1">'setlang'</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">babel_lang</span> <span class="o">=</span> <span class="n">map_lang</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">)</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">babel_lang</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">)))</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: language (</span><span class="si">%s</span><span class="s2">) is unknown by babel"</span> <span class="o">%</span> <span class="p">(</span><span class="n">babel_lang</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="c1"># Language (e.g. 'en' or 'de') from https://www.bing.com/account/general</span>
|
||||||
|
<span class="c1"># is converted by bing to 'en-us' or 'de-de'. But only if there is not</span>
|
||||||
|
<span class="c1"># already a '-' delemitter in the language. For instance 'pt-PT' --></span>
|
||||||
|
<span class="c1"># 'pt-pt' and 'pt-br' --> 'pt-br'</span>
|
||||||
|
<span class="n">bing_ui_lang</span> <span class="o">=</span> <span class="n">eng_lang</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'-'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">bing_ui_lang</span><span class="p">:</span>
|
||||||
|
<span class="n">bing_ui_lang</span> <span class="o">=</span> <span class="n">bing_ui_lang</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">bing_ui_lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">bing_ui_lang</span><span class="p">,</span> <span class="n">bing_ui_lang</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">bing_ui_lang</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"CONFLICT: babel </span><span class="si">{</span><span class="n">sxng_tag</span><span class="si">}</span><span class="s2"> --> </span><span class="si">{</span><span class="n">conflict</span><span class="si">}</span><span class="s2">, </span><span class="si">{</span><span class="n">bing_ui_lang</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">bing_ui_lang</span>
|
||||||
|
|
||||||
|
<span class="c1"># regions (aka "market codes")</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="s1">'zh-CN'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'zh-cn'</span>
|
||||||
|
|
||||||
|
<span class="n">map_market_codes</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'zh-hk'</span><span class="p">:</span> <span class="s1">'en-hk'</span><span class="p">,</span> <span class="c1"># not sure why, but at M$ this is the market code for Hongkong</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="k">for</span> <span class="n">href</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[@id="region-section-content"]//div[@class="regionItem"]/a/@href'</span><span class="p">):</span>
|
||||||
|
<span class="n">cc_tag</span> <span class="o">=</span> <span class="n">parse_qs</span><span class="p">(</span><span class="n">urlparse</span><span class="p">(</span><span class="n">href</span><span class="p">)</span><span class="o">.</span><span class="n">query</span><span class="p">)[</span><span class="s1">'cc'</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">cc_tag</span> <span class="o">==</span> <span class="s1">'clear'</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="n">cc_tag</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="c1"># add market codes from official languages of the country ..</span>
|
||||||
|
<span class="k">for</span> <span class="n">lang_tag</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">cc_tag</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">lang_tag</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
||||||
|
<span class="c1"># print("ignore lang: %s <-- %s" % (cc_tag, lang_tag))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">lang_tag</span> <span class="o">=</span> <span class="n">lang_tag</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># zh_Hant --> zh</span>
|
||||||
|
<span class="n">market_code</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">lang_tag</span><span class="si">}</span><span class="s2">-</span><span class="si">{</span><span class="n">cc_tag</span><span class="si">}</span><span class="s2">"</span> <span class="c1"># zh-tw</span>
|
||||||
|
|
||||||
|
<span class="n">market_code</span> <span class="o">=</span> <span class="n">map_market_codes</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">market_code</span><span class="p">,</span> <span class="n">market_code</span><span class="p">)</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="s1">'</span><span class="si">%s</span><span class="s1">_</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">lang_tag</span><span class="p">,</span> <span class="n">cc_tag</span><span class="o">.</span><span class="n">upper</span><span class="p">())))</span>
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">market_code</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">market_code</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">market_code</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
210
_modules/searx/engines/bing_images.html
Normal file
@@ -0,0 +1,210 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.bing_images — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.bing_images</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.bing_images</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Bing-Images: description see :py:obj:`searx.engines.bing`."""</span>
|
||||||
|
<span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">set_bing_cookies</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.bing.com/images'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q182496'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://www.microsoft.com/en-us/bing/apis/bing-image-search-api'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'images'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://www.bing.com/images/async'</span>
|
||||||
|
<span class="sd">"""Bing (Images) search URL"""</span>
|
||||||
|
|
||||||
|
<span class="n">time_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'day'</span><span class="p">:</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span><span class="p">,</span>
|
||||||
|
<span class="s1">'week'</span><span class="p">:</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
|
||||||
|
<span class="s1">'month'</span><span class="p">:</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">31</span><span class="p">,</span>
|
||||||
|
<span class="s1">'year'</span><span class="p">:</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">365</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_images.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Assemble a Bing-Image request."""</span>
|
||||||
|
|
||||||
|
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="s1">'en'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># build URL query</span>
|
||||||
|
<span class="c1"># - example: https://www.bing.com/images/async?q=foo&async=content&first=1&count=35</span>
|
||||||
|
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'async'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># to simplify the page count lets use the default of 35 images per page</span>
|
||||||
|
<span class="s1">'first'</span><span class="p">:</span> <span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'pageno'</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">35</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="s1">'count'</span><span class="p">:</span> <span class="mi">35</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># time range</span>
|
||||||
|
<span class="c1"># - example: one year (525600 minutes) 'qft=+filterui:age-lt525600'</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]:</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'qft'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'filterui:age-lt</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">time_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">'?'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_images.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get response from Bing-Images"""</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//ul[contains(@class, "dgControl_list")]/li'</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">metadata</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//a[@class="iusc"]/@m'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">metadata</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">metadata</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//a[@class="iusc"]/@m'</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//div[@class="infnmpt"]//a/text()'</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="n">img_format</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//div[@class="imgpt"]/div/span/text()'</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">" · "</span><span class="p">)</span>
|
||||||
|
<span class="n">source</span> <span class="o">=</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//div[@class="imgpt"]//div[@class="lnkw"]//a/text()'</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'images.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">metadata</span><span class="p">[</span><span class="s1">'purl'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'thumbnail_src'</span><span class="p">:</span> <span class="n">metadata</span><span class="p">[</span><span class="s1">'turl'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">metadata</span><span class="p">[</span><span class="s1">'murl'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'desc'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'source'</span><span class="p">:</span> <span class="n">source</span><span class="p">,</span>
|
||||||
|
<span class="s1">'resolution'</span><span class="p">:</span> <span class="n">img_format</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
|
||||||
|
<span class="s1">'img_format'</span><span class="p">:</span> <span class="n">img_format</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">img_format</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">2</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
268
_modules/searx/engines/bing_news.html
Normal file
@@ -0,0 +1,268 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.bing_news — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.bing_news</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.bing_news</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Bing-News: description see :py:obj:`searx.engines.bing`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> Bing News is *different* in some ways!</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">set_bing_cookies</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.bing.com/news'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q2878637'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://www.microsoft.com/en-us/bing/apis/bing-news-search-api'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'RSS'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'news'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="sd">"""If go through the pages and there are actually no new results for another</span>
|
||||||
|
<span class="sd">page, then bing returns the results from the last page again."""</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'day'</span><span class="p">:</span> <span class="s1">'interval="4"'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'week'</span><span class="p">:</span> <span class="s1">'interval="7"'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'month'</span><span class="p">:</span> <span class="s1">'interval="9"'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""A string '4' means *last hour*. We use *last hour* for ``day`` here since the</span>
|
||||||
|
<span class="sd">difference of *last day* and *last week* in the result list is just marginally.</span>
|
||||||
|
<span class="sd">Bing does not have news range ``year`` / we use ``month`` instead."""</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://www.bing.com/news/infinitescrollajax'</span>
|
||||||
|
<span class="sd">"""Bing (News) search URL"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_news.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Assemble a Bing-News request."""</span>
|
||||||
|
|
||||||
|
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="s1">'en'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># build URL query</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># example: https://www.bing.com/news/infinitescrollajax?q=london&first=1</span>
|
||||||
|
|
||||||
|
<span class="n">page</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'pageno'</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span>
|
||||||
|
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'InfiniteScroll'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="c1"># to simplify the page count lets use the default of 10 images per page</span>
|
||||||
|
<span class="s1">'first'</span><span class="p">:</span> <span class="n">page</span> <span class="o">*</span> <span class="mi">10</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="s1">'SFX'</span><span class="p">:</span> <span class="n">page</span><span class="p">,</span>
|
||||||
|
<span class="s1">'form'</span><span class="p">:</span> <span class="s1">'PTFTNR'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'setlang'</span><span class="p">:</span> <span class="n">engine_region</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
|
||||||
|
<span class="s1">'cc'</span><span class="p">:</span> <span class="n">engine_region</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]:</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'qft'</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">],</span> <span class="s1">'interval="9"'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">'?'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_news.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get response from Bing-Video"""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">newsitem</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class, "newsitem")]'</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">link</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">newsitem</span><span class="p">,</span> <span class="s1">'.//a[@class="title"]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">link</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">link</span><span class="o">.</span><span class="n">attrib</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'href'</span><span class="p">)</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">link</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">newsitem</span><span class="p">,</span> <span class="s1">'.//div[@class="snippet"]'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">metadata</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">source</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">newsitem</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "source")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">source</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="s1">'.//span[@aria-label]/@aria-label'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
|
||||||
|
<span class="c1"># eval_xpath_getindex(source, './/a', 0, None),</span>
|
||||||
|
<span class="c1"># eval_xpath_getindex(source, './div/span', 3, None),</span>
|
||||||
|
<span class="n">link</span><span class="o">.</span><span class="n">attrib</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'data-author'</span><span class="p">),</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">item</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">t</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">t</span> <span class="ow">and</span> <span class="n">t</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
|
||||||
|
<span class="n">metadata</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
|
||||||
|
<span class="n">metadata</span> <span class="o">=</span> <span class="s1">' | '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">imagelink</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">newsitem</span><span class="p">,</span> <span class="s1">'.//a[@class="imagelink"]//img'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">imagelink</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">imagelink</span><span class="o">.</span><span class="n">attrib</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'src'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"https://www.bing.com"</span><span class="p">):</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="s1">'https://www.bing.com/'</span> <span class="o">+</span> <span class="n">thumbnail</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="s1">'metadata'</span><span class="p">:</span> <span class="n">metadata</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_news.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages and regions from Bing-News."""</span>
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="k">as</span> <span class="n">_f</span>
|
||||||
|
|
||||||
|
<span class="n">_f</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># fix market codes not known by bing news:</span>
|
||||||
|
|
||||||
|
<span class="c1"># In bing the market code 'zh-cn' exists, but there is no 'news' category in</span>
|
||||||
|
<span class="c1"># bing for this market. Alternatively we use the the market code from Honk</span>
|
||||||
|
<span class="c1"># Kong. Even if this is not correct, it is better than having no hits at</span>
|
||||||
|
<span class="c1"># all, or sending false queries to bing that could raise the suspicion of a</span>
|
||||||
|
<span class="c1"># bot.</span>
|
||||||
|
|
||||||
|
<span class="c1"># HINT: 'en-hk' is the region code it does not indicate the language en!!</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="s1">'zh-CN'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'en-hk'</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
202
_modules/searx/engines/bing_videos.html
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.bing_videos — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.bing_videos</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.bing_videos</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
<span class="sd">"""Bing-Videos: description see :py:obj:`searx.engines.bing`."""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">set_bing_cookies</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.bing_images</span><span class="w"> </span><span class="kn">import</span> <span class="n">time_map</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.bing.com/videos'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q4914152'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://www.microsoft.com/en-us/bing/apis/bing-video-search-api'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'videos'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://www.bing.com/videos/asyncv2'</span>
|
||||||
|
<span class="sd">"""Bing (Videos) async search URL."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_videos.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Assemble a Bing-Video request."""</span>
|
||||||
|
|
||||||
|
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="s1">'en'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">set_bing_cookies</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">engine_language</span><span class="p">,</span> <span class="n">engine_region</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># build URL query</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># example: https://www.bing.com/videos/asyncv2?q=foo&async=content&first=1&count=35</span>
|
||||||
|
|
||||||
|
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'async'</span><span class="p">:</span> <span class="s1">'content'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># to simplify the page count lets use the default of 35 images per page</span>
|
||||||
|
<span class="s1">'first'</span><span class="p">:</span> <span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'pageno'</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">35</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="s1">'count'</span><span class="p">:</span> <span class="mi">35</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># time range</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># example: one week (10080 minutes) '&qft= filterui:videoage-lt10080' '&form=VRFLTR'</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]:</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'form'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'VRFLTR'</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'qft'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">' filterui:videoage-lt</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">time_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">'?'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/bing.html#searx.engines.bing_videos.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get response from Bing-Video"""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//div[@class="dg_u"]//div[contains(@id, "mc_vtvc_video")]'</span><span class="p">):</span>
|
||||||
|
<span class="n">metadata</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//div[@class="vrhdata"]/@vrhm'</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="n">info</span> <span class="o">=</span> <span class="s1">' - '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//div[@class="mc_vtvc_meta_block"]//span/text()'</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="s1">'</span><span class="si">{0}</span><span class="s1"> - </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s1">'du'</span><span class="p">],</span> <span class="n">info</span><span class="p">)</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//div[contains(@class, "mc_vtvc_th")]//img/@src'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">metadata</span><span class="p">[</span><span class="s1">'murl'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'vt'</span><span class="p">,</span> <span class="s1">''</span><span class="p">),</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'videos.html'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
596
_modules/searx/engines/brave.html
Normal file
@@ -0,0 +1,596 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.brave — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.brave</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.brave</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Brave supports the categories listed in :py:obj:`brave_category` (General,</span>
|
||||||
|
<span class="sd">news, videos, images). The support of :py:obj:`paging` and :py:obj:`time range</span>
|
||||||
|
<span class="sd"><time_range_support>` is limited (see remarks).</span>
|
||||||
|
|
||||||
|
<span class="sd">Configured ``brave`` engines:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: brave</span>
|
||||||
|
<span class="sd"> engine: brave</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> brave_category: search</span>
|
||||||
|
<span class="sd"> time_range_support: true</span>
|
||||||
|
<span class="sd"> paging: true</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: brave.images</span>
|
||||||
|
<span class="sd"> engine: brave</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> brave_category: images</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: brave.videos</span>
|
||||||
|
<span class="sd"> engine: brave</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> brave_category: videos</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: brave.news</span>
|
||||||
|
<span class="sd"> engine: brave</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> brave_category: news</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: brave.goggles</span>
|
||||||
|
<span class="sd"> time_range_support: true</span>
|
||||||
|
<span class="sd"> paging: true</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> brave_category: goggles</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">.. _brave regions:</span>
|
||||||
|
|
||||||
|
<span class="sd">Brave regions</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">Brave uses two-digit tags for the regions like ``ca`` while SearXNG deals with</span>
|
||||||
|
<span class="sd">locales. To get a mapping, all *officiat de-facto* languages of the Brave</span>
|
||||||
|
<span class="sd">region are mapped to regions in SearXNG (see :py:obj:`babel</span>
|
||||||
|
<span class="sd"><babel.languages.get_official_languages>`):</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> "regions": {</span>
|
||||||
|
<span class="sd"> ..</span>
|
||||||
|
<span class="sd"> "en-CA": "ca",</span>
|
||||||
|
<span class="sd"> "fr-CA": "ca",</span>
|
||||||
|
<span class="sd"> ..</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">.. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> The language (aka region) support of Brave's index is limited to very basic</span>
|
||||||
|
<span class="sd"> languages. The search results for languages like Chinese or Arabic are of</span>
|
||||||
|
<span class="sd"> low quality.</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">.. _brave googles:</span>
|
||||||
|
|
||||||
|
<span class="sd">Brave Goggles</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _list of Goggles: https://search.brave.com/goggles/discover</span>
|
||||||
|
<span class="sd">.. _Goggles Whitepaper: https://brave.com/static-assets/files/goggles.pdf</span>
|
||||||
|
<span class="sd">.. _Goggles Quickstart: https://github.com/brave/goggles-quickstart</span>
|
||||||
|
|
||||||
|
<span class="sd">Goggles allow you to choose, alter, or extend the ranking of Brave Search</span>
|
||||||
|
<span class="sd">results (`Goggles Whitepaper`_). Goggles are openly developed by the community</span>
|
||||||
|
<span class="sd">of Brave Search users.</span>
|
||||||
|
|
||||||
|
<span class="sd">Select from the `list of Goggles`_ people have published, or create your own</span>
|
||||||
|
<span class="sd">(`Goggles Quickstart`_).</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">.. _brave languages:</span>
|
||||||
|
|
||||||
|
<span class="sd">Brave languages</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">Brave's language support is limited to the UI (menus, area local notations,</span>
|
||||||
|
<span class="sd">etc). Brave's index only seems to support a locale, but it does not seem to</span>
|
||||||
|
<span class="sd">support any languages in its index. The choice of available languages is very</span>
|
||||||
|
<span class="sd">small (and its not clear to me where the difference in UI is when switching</span>
|
||||||
|
<span class="sd">from en-us to en-ca or en-gb).</span>
|
||||||
|
|
||||||
|
<span class="sd">In the :py:obj:`EngineTraits object <searx.enginelib.traits.EngineTraits>` the</span>
|
||||||
|
<span class="sd">UI languages are stored in a custom field named ``ui_lang``:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> "custom": {</span>
|
||||||
|
<span class="sd"> "ui_lang": {</span>
|
||||||
|
<span class="sd"> "ca": "ca",</span>
|
||||||
|
<span class="sd"> "de-DE": "de-de",</span>
|
||||||
|
<span class="sd"> "en-CA": "en-ca",</span>
|
||||||
|
<span class="sd"> "en-GB": "en-gb",</span>
|
||||||
|
<span class="sd"> "en-US": "en-us",</span>
|
||||||
|
<span class="sd"> "es": "es",</span>
|
||||||
|
<span class="sd"> "fr-CA": "fr-ca",</span>
|
||||||
|
<span class="sd"> "fr-FR": "fr-fr",</span>
|
||||||
|
<span class="sd"> "ja-JP": "ja-jp",</span>
|
||||||
|
<span class="sd"> "pt-BR": "pt-br",</span>
|
||||||
|
<span class="sd"> "sq-AL": "sq-al"</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
<span class="sd"> },</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">urlencode</span><span class="p">,</span>
|
||||||
|
<span class="n">urlparse</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil</span><span class="w"> </span><span class="kn">import</span> <span class="n">parser</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">locales</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||||
|
<span class="n">js_obj_str_to_python</span><span class="p">,</span>
|
||||||
|
<span class="n">js_obj_str_to_json_str</span><span class="p">,</span>
|
||||||
|
<span class="n">get_embeded_stream_url</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://search.brave.com/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q22906900"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"HTML"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://search.brave.com/"</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">brave_category</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"search"</span><span class="p">,</span> <span class="s2">"videos"</span><span class="p">,</span> <span class="s2">"images"</span><span class="p">,</span> <span class="s2">"news"</span><span class="p">,</span> <span class="s2">"goggles"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"search"</span>
|
||||||
|
<span class="sd">"""Brave supports common web-search, videos, images, news, and goggles search.</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``search``: Common WEB search</span>
|
||||||
|
<span class="sd">- ``videos``: search for videos</span>
|
||||||
|
<span class="sd">- ``images``: search for images</span>
|
||||||
|
<span class="sd">- ``news``: search for news</span>
|
||||||
|
<span class="sd">- ``goggles``: Common WEB search with custom rules, requires a :py:obj:`Goggles` URL.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">Goggles</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""This should be a URL ending in ``.goggle``"""</span>
|
||||||
|
|
||||||
|
<span class="n">brave_spellcheck</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">"""Brave supports some kind of spell checking. When activated, Brave tries to</span>
|
||||||
|
<span class="sd">fix typos, e.g. it searches for ``food`` when the user queries for ``fooh``. In</span>
|
||||||
|
<span class="sd">the UI of Brave the user gets warned about this, since we can not warn the user</span>
|
||||||
|
<span class="sd">in SearXNG, the spellchecking is disabled by default.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">"""Brave only supports paging in :py:obj:`brave_category` ``search`` (UI</span>
|
||||||
|
<span class="sd">category All) and in the goggles category."""</span>
|
||||||
|
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">10</span>
|
||||||
|
<span class="sd">"""Tested 9 pages maximum (``&offset=8``), to be save max is set to 10. Trying</span>
|
||||||
|
<span class="sd">to do more won't return any result and you will most likely be flagged as a bot.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch_map</span> <span class="o">=</span> <span class="p">{</span><span class="mi">2</span><span class="p">:</span> <span class="s2">"strict"</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s2">"moderate"</span><span class="p">,</span> <span class="mi">0</span><span class="p">:</span> <span class="s2">"off"</span><span class="p">}</span> <span class="c1"># cookie: safesearch=off</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">"""Brave only supports time-range in :py:obj:`brave_category` ``search`` (UI</span>
|
||||||
|
<span class="sd">category All) and in the goggles category."""</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"day"</span><span class="p">:</span> <span class="s2">"pd"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"week"</span><span class="p">:</span> <span class="s2">"pw"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"month"</span><span class="p">:</span> <span class="s2">"pm"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"year"</span><span class="p">:</span> <span class="s2">"py"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">args</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"q"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"source"</span><span class="p">:</span> <span class="s2">"web"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="k">if</span> <span class="n">brave_spellcheck</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s2">"spellcheck"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"1"</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">brave_category</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"search"</span><span class="p">,</span> <span class="s2">"goggles"</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"pageno"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s2">"offset"</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"pageno"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>
|
||||||
|
<span class="k">if</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"time_range"</span><span class="p">]):</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s2">"tf"</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"time_range"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">brave_category</span> <span class="o">==</span> <span class="s2">"goggles"</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s2">"goggles_id"</span><span class="p">]</span> <span class="o">=</span> <span class="n">Goggles</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">][</span><span class="s2">"Accept-Encoding"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"gzip, deflate"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}{</span><span class="n">brave_category</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"url </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="c1"># set properties in the cookies</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">][</span><span class="s2">"safesearch"</span><span class="p">]</span> <span class="o">=</span> <span class="n">safesearch_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"safesearch"</span><span class="p">],</span> <span class="s2">"off"</span><span class="p">)</span>
|
||||||
|
<span class="c1"># the useLocation is IP based, we use cookie "country" for the region</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">][</span><span class="s2">"useLocation"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"0"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">][</span><span class="s2">"summarizer"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"0"</span>
|
||||||
|
|
||||||
|
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"searxng_locale"</span><span class="p">],</span> <span class="s2">"all"</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_region</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"-"</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="n">ui_lang</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">get_engine_locale</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"searxng_locale"</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ui_lang"</span><span class="p">],</span> <span class="s2">"en-us"</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">][</span><span class="s2">"ui_lang"</span><span class="p">]</span> <span class="o">=</span> <span class="n">ui_lang</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"cookies </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_extract_published_date</span><span class="p">(</span><span class="n">published_date_raw</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">published_date_raw</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">published_date_raw</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">parser</span><span class="o">.</span><span class="n">ParserError</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">extract_json_data</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># Example script source containing the data:</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># kit.start(app, element, {</span>
|
||||||
|
<span class="c1"># node_ids: [0, 19],</span>
|
||||||
|
<span class="c1"># data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]</span>
|
||||||
|
<span class="c1"># ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^</span>
|
||||||
|
<span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="p">[</span><span class="n">text</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">"<script"</span><span class="p">)</span> <span class="p">:</span> <span class="n">text</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">"</script"</span><span class="p">)]</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">text</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"can't find JS/JSON data in the given text"</span><span class="p">)</span>
|
||||||
|
<span class="n">start</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">"data: [{"</span><span class="p">)</span>
|
||||||
|
<span class="n">end</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">rindex</span><span class="p">(</span><span class="s2">"}}]"</span><span class="p">)</span>
|
||||||
|
<span class="n">js_obj_str</span> <span class="o">=</span> <span class="n">text</span><span class="p">[</span><span class="n">start</span><span class="p">:</span><span class="n">end</span><span class="p">]</span>
|
||||||
|
<span class="n">js_obj_str</span> <span class="o">=</span> <span class="s2">"{"</span> <span class="o">+</span> <span class="n">js_obj_str</span> <span class="o">+</span> <span class="s2">"}}]}"</span>
|
||||||
|
<span class="c1"># js_obj_str = js_obj_str.replace("\xa0", "") # remove ASCII for &nbsp;</span>
|
||||||
|
<span class="c1"># js_obj_str = js_obj_str.replace(r"\u003C", "<").replace(r"\u003c", "<") # fix broken HTML tags in strings</span>
|
||||||
|
<span class="n">json_str</span> <span class="o">=</span> <span class="n">js_obj_str_to_json_str</span><span class="p">(</span><span class="n">js_obj_str</span><span class="p">)</span>
|
||||||
|
<span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">json_str</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">brave_category</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'search'</span><span class="p">,</span> <span class="s1">'goggles'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">_parse_search</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">brave_category</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'news'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">_parse_news</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Example script source containing the data:</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># kit.start(app, element, {</span>
|
||||||
|
<span class="c1"># node_ids: [0, 19],</span>
|
||||||
|
<span class="c1"># data: [{type:"data",data: .... ["q","goggles_id"],route:1,url:1}}]</span>
|
||||||
|
<span class="c1"># ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^</span>
|
||||||
|
<span class="n">json_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_json_data</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="n">json_resp</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="mi">1</span><span class="p">][</span><span class="s2">"data"</span><span class="p">][</span><span class="s1">'body'</span><span class="p">][</span><span class="s1">'response'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">brave_category</span> <span class="o">==</span> <span class="s1">'images'</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">_parse_images</span><span class="p">(</span><span class="n">json_resp</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">brave_category</span> <span class="o">==</span> <span class="s1">'videos'</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">_parse_videos</span><span class="p">(</span><span class="n">json_resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unsupported brave category: </span><span class="si">{</span><span class="n">brave_category</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_search</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[contains(@class, 'snippet ')]"</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//a/@href"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">title_tag</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//div[contains(@class, 'title')]"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">title_tag</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">netloc</span><span class="p">:</span> <span class="c1"># partial url likely means it's an ad</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">pub_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># there are other classes like 'site-name-content' we don't want to match,</span>
|
||||||
|
<span class="c1"># however only using contains(@class, 'content') would e.g. also match `site-name-content`</span>
|
||||||
|
<span class="c1"># thus, we explicitly also require the spaces as class separator</span>
|
||||||
|
<span class="n">_content</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//div[contains(concat(' ', @class, ' '), ' content ')]"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">_content</span><span class="p">):</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">_content</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">_pub_date</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">_content</span><span class="p">,</span> <span class="s2">".//span[contains(@class, 't-secondary')]"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">_pub_date</span><span class="p">:</span>
|
||||||
|
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">_extract_published_date</span><span class="p">(</span><span class="n">_pub_date</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="n">_pub_date</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"- </span><span class="se">\n\t</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//a[contains(@class, 'thumbnail')]//img/@src"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">""</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span>
|
||||||
|
<span class="n">template</span><span class="o">=</span><span class="s2">"default.html"</span><span class="p">,</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">title_tag</span><span class="p">),</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="n">publishedDate</span><span class="o">=</span><span class="n">pub_date</span><span class="p">,</span>
|
||||||
|
<span class="n">thumbnail</span><span class="o">=</span><span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">video_tag</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span>
|
||||||
|
<span class="n">result</span><span class="p">,</span> <span class="s2">".//div[contains(@class, 'video-snippet') and @data-macro='video']"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="p">[]</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">video_tag</span><span class="p">):</span>
|
||||||
|
<span class="c1"># In my tests a video tag in the WEB search was most often not a</span>
|
||||||
|
<span class="c1"># video, except the ones from youtube ..</span>
|
||||||
|
<span class="n">iframe_src</span> <span class="o">=</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">iframe_src</span><span class="p">:</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"iframe_src"</span><span class="p">]</span> <span class="o">=</span> <span class="n">iframe_src</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"template"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"videos.html"</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//a[contains(@class, 'related-query')]"</span><span class="p">):</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)}))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_news</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[contains(@class, 'results')]//div[@data-type='news']"</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//a[contains(@class, 'result-header')]/@href"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//span[contains(@class, 'snippet-title')]"</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//p[contains(@class, 'desc')]"</span><span class="p">)</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//div[contains(@class, 'image-wrapper')]//img/@src"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s2">""</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span>
|
||||||
|
<span class="n">template</span><span class="o">=</span><span class="s2">"default.html"</span><span class="p">,</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">title</span><span class="p">),</span>
|
||||||
|
<span class="n">thumbnail</span><span class="o">=</span><span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">content</span><span class="p">),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_images</span><span class="p">(</span><span class="n">json_resp</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_resp</span><span class="p">[</span><span class="s2">"results"</span><span class="p">]:</span>
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span>
|
||||||
|
<span class="n">template</span><span class="o">=</span><span class="s2">"images.html"</span><span class="p">,</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"url"</span><span class="p">],</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"title"</span><span class="p">],</span>
|
||||||
|
<span class="n">source</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"source"</span><span class="p">],</span>
|
||||||
|
<span class="n">img_src</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"properties"</span><span class="p">][</span><span class="s2">"url"</span><span class="p">],</span>
|
||||||
|
<span class="n">thumbnail_src</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"thumbnail"</span><span class="p">][</span><span class="s2">"src"</span><span class="p">],</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_videos</span><span class="p">(</span><span class="n">json_resp</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_resp</span><span class="p">[</span><span class="s2">"results"</span><span class="p">]:</span>
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span>
|
||||||
|
<span class="n">template</span><span class="o">=</span><span class="s2">"videos.html"</span><span class="p">,</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"url"</span><span class="p">],</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"title"</span><span class="p">],</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"description"</span><span class="p">],</span>
|
||||||
|
<span class="n">length</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"video"</span><span class="p">][</span><span class="s2">"duration"</span><span class="p">],</span>
|
||||||
|
<span class="n">duration</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"video"</span><span class="p">][</span><span class="s2">"duration"</span><span class="p">],</span>
|
||||||
|
<span class="n">publishedDate</span><span class="o">=</span><span class="n">_extract_published_date</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"age"</span><span class="p">]),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="p">[</span><span class="s2">"thumbnail"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"thumbnail"</span><span class="p">]</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"thumbnail"</span><span class="p">][</span><span class="s2">"src"</span><span class="p">]</span>
|
||||||
|
<span class="n">iframe_src</span> <span class="o">=</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"url"</span><span class="p">])</span>
|
||||||
|
<span class="k">if</span> <span class="n">iframe_src</span><span class="p">:</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"iframe_src"</span><span class="p">]</span> <span class="o">=</span> <span class="n">iframe_src</span>
|
||||||
|
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/brave.html#searx.engines.brave.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch :ref:`languages <brave languages>` and :ref:`regions <brave</span>
|
||||||
|
<span class="sd"> regions>` from Brave."""</span>
|
||||||
|
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel, too-many-branches</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.languages</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">region_tag</span><span class="p">,</span> <span class="n">language_tag</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ui_lang"</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="n">lang_map</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'no'</span><span class="p">:</span> <span class="s1">'nb'</span><span class="p">}</span> <span class="c1"># norway</span>
|
||||||
|
|
||||||
|
<span class="c1"># languages (UI)</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">'https://search.brave.com/settings'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from Brave is not OK."</span><span class="p">)</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">option</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s2">"//section//option[@value='en-us']/../option"</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">ui_lang</span> <span class="o">=</span> <span class="n">option</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">l</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">ui_lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">"-"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">l</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">ui_lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">"-"</span><span class="p">))</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">ui_lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">"-"</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: can't determine babel locale of Brave's (UI) language </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">ui_lang</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ui_lang"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">ui_lang</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">ui_lang</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ui_lang"</span><span class="p">][</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">ui_lang</span>
|
||||||
|
|
||||||
|
<span class="c1"># search regions of brave</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s2">"https://cdn.search.brave.com/serp/v2/_app/immutable/chunks/parameters.734c106a.js"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from Brave is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">country_js</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">[</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">"options:{all"</span><span class="p">)</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="s2">"options:"</span><span class="p">)</span> <span class="p">:]</span>
|
||||||
|
<span class="n">country_js</span> <span class="o">=</span> <span class="n">country_js</span><span class="p">[:</span> <span class="n">country_js</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">"},k={default"</span><span class="p">)]</span>
|
||||||
|
<span class="n">country_tags</span> <span class="o">=</span> <span class="n">js_obj_str_to_python</span><span class="p">(</span><span class="n">country_js</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">country_tags</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="n">k</span> <span class="o">==</span> <span class="s2">"all"</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s2">"all"</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">country_tag</span> <span class="o">=</span> <span class="n">v</span><span class="p">[</span><span class="s2">"value"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># add official languages of the country ..</span>
|
||||||
|
<span class="k">for</span> <span class="n">lang_tag</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">country_tag</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="n">lang_tag</span> <span class="o">=</span> <span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang_tag</span><span class="p">,</span> <span class="n">lang_tag</span><span class="p">)</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">_</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">lang_tag</span><span class="p">,</span> <span class="n">country_tag</span><span class="o">.</span><span class="n">upper</span><span class="p">())))</span>
|
||||||
|
<span class="c1"># print("%-20s: %s <-- %s" % (v["label"], country_tag, sxng_tag))</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">country_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">country_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">country_tag</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
354
_modules/searx/engines/command.html
Normal file
@@ -0,0 +1,354 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.command — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.command</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.command</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""With *command engines* administrators can run engines to integrate arbitrary</span>
|
||||||
|
<span class="sd">shell commands.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. attention::</span>
|
||||||
|
|
||||||
|
<span class="sd"> When creating and enabling a ``command`` engine on a public instance, you</span>
|
||||||
|
<span class="sd"> must be careful to avoid leaking private data.</span>
|
||||||
|
|
||||||
|
<span class="sd">The easiest solution is to limit the access by setting ``tokens`` as described</span>
|
||||||
|
<span class="sd">in section :ref:`private engines`. The engine base is flexible. Only your</span>
|
||||||
|
<span class="sd">imagination can limit the power of this engine (and maybe security concerns).</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The following options are available:</span>
|
||||||
|
|
||||||
|
<span class="sd">``command``:</span>
|
||||||
|
<span class="sd"> A comma separated list of the elements of the command. A special token</span>
|
||||||
|
<span class="sd"> ``{{QUERY}}`` tells where to put the search terms of the user. Example:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> ['ls', '-l', '-h', '{{QUERY}}']</span>
|
||||||
|
|
||||||
|
<span class="sd">``delimiter``:</span>
|
||||||
|
<span class="sd"> A mapping containing a delimiter ``char`` and the *titles* of each element in</span>
|
||||||
|
<span class="sd"> ``keys``.</span>
|
||||||
|
|
||||||
|
<span class="sd">``parse_regex``:</span>
|
||||||
|
<span class="sd"> A dict containing the regular expressions for each result key.</span>
|
||||||
|
|
||||||
|
<span class="sd">``query_type``:</span>
|
||||||
|
|
||||||
|
<span class="sd"> The expected type of user search terms. Possible values: ``path`` and</span>
|
||||||
|
<span class="sd"> ``enum``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``path``:</span>
|
||||||
|
<span class="sd"> Checks if the user provided path is inside the working directory. If not,</span>
|
||||||
|
<span class="sd"> the query is not executed.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``enum``:</span>
|
||||||
|
<span class="sd"> Is a list of allowed search terms. If the user submits something which is</span>
|
||||||
|
<span class="sd"> not included in the list, the query returns an error.</span>
|
||||||
|
|
||||||
|
<span class="sd">``query_enum``:</span>
|
||||||
|
<span class="sd"> A list containing allowed search terms if ``query_type`` is set to ``enum``.</span>
|
||||||
|
|
||||||
|
<span class="sd">``working_dir``:</span>
|
||||||
|
<span class="sd"> The directory where the command has to be executed. Default: ``./``.</span>
|
||||||
|
|
||||||
|
<span class="sd">``result_separator``:</span>
|
||||||
|
<span class="sd"> The character that separates results. Default: ``\\n``.</span>
|
||||||
|
|
||||||
|
<span class="sd">Example</span>
|
||||||
|
<span class="sd">=======</span>
|
||||||
|
|
||||||
|
<span class="sd">The example engine below can be used to find files with a specific name in the</span>
|
||||||
|
<span class="sd">configured working directory:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: find</span>
|
||||||
|
<span class="sd"> engine: command</span>
|
||||||
|
<span class="sd"> command: ['find', '.', '-name', '{{QUERY}}']</span>
|
||||||
|
<span class="sd"> query_type: path</span>
|
||||||
|
<span class="sd"> shortcut: fnd</span>
|
||||||
|
<span class="sd"> delimiter:</span>
|
||||||
|
<span class="sd"> chars: ' '</span>
|
||||||
|
<span class="sd"> keys: ['line']</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">os.path</span><span class="w"> </span><span class="kn">import</span> <span class="n">expanduser</span><span class="p">,</span> <span class="n">isabs</span><span class="p">,</span> <span class="n">realpath</span><span class="p">,</span> <span class="n">commonprefix</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">shlex</span><span class="w"> </span><span class="kn">import</span> <span class="n">split</span> <span class="k">as</span> <span class="n">shlex_split</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">subprocess</span><span class="w"> </span><span class="kn">import</span> <span class="n">Popen</span><span class="p">,</span> <span class="n">PIPE</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">threading</span><span class="w"> </span><span class="kn">import</span> <span class="n">Thread</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">engine_type</span> <span class="o">=</span> <span class="s1">'offline'</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">command</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">delimiter</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">parse_regex</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">query_type</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="n">query_enum</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">environment_variables</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">working_dir</span> <span class="o">=</span> <span class="n">realpath</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
|
||||||
|
<span class="n">result_separator</span> <span class="o">=</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span>
|
||||||
|
<span class="n">timeout</span> <span class="o">=</span> <span class="mf">4.0</span>
|
||||||
|
|
||||||
|
<span class="n">_command_logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'command'</span><span class="p">)</span>
|
||||||
|
<span class="n">_compiled_parse_regex</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span>
|
||||||
|
<span class="n">check_parsing_options</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'command'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'engine command : missing configuration key: command'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">global</span> <span class="n">command</span><span class="p">,</span> <span class="n">working_dir</span><span class="p">,</span> <span class="n">delimiter</span><span class="p">,</span> <span class="n">parse_regex</span><span class="p">,</span> <span class="n">environment_variables</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
|
||||||
|
<span class="n">command</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">'command'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'working_dir'</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="n">working_dir</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">'working_dir'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">isabs</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">[</span><span class="s1">'working_dir'</span><span class="p">]):</span>
|
||||||
|
<span class="n">working_dir</span> <span class="o">=</span> <span class="n">realpath</span><span class="p">(</span><span class="n">working_dir</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'parse_regex'</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="n">parse_regex</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">'parse_regex'</span><span class="p">]</span>
|
||||||
|
<span class="k">for</span> <span class="n">result_key</span><span class="p">,</span> <span class="n">regex</span> <span class="ow">in</span> <span class="n">parse_regex</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">_compiled_parse_regex</span><span class="p">[</span><span class="n">result_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">regex</span><span class="p">,</span> <span class="n">flags</span><span class="o">=</span><span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'delimiter'</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="n">delimiter</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">'delimiter'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'environment_variables'</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="n">environment_variables</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">'environment_variables'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">search</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">cmd</span> <span class="o">=</span> <span class="n">_get_command_to_run</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">cmd</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
<span class="n">reader_thread</span> <span class="o">=</span> <span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">_get_results_from_process</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">res</span><span class="p">,</span> <span class="n">cmd</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]))</span>
|
||||||
|
<span class="n">reader_thread</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
||||||
|
<span class="n">reader_thread</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_get_command_to_run</span><span class="p">(</span><span class="n">query</span><span class="p">):</span>
|
||||||
|
<span class="n">params</span> <span class="o">=</span> <span class="n">shlex_split</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
|
||||||
|
<span class="n">__check_query_params</span><span class="p">(</span><span class="n">params</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">cmd</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">command</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">c</span> <span class="o">==</span> <span class="s1">'{{QUERY}}'</span><span class="p">:</span>
|
||||||
|
<span class="n">cmd</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">params</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">cmd</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">cmd</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_get_results_from_process</span><span class="p">(</span><span class="n">res</span><span class="p">:</span> <span class="n">EngineResults</span><span class="p">,</span> <span class="n">cmd</span><span class="p">,</span> <span class="n">pageno</span><span class="p">):</span>
|
||||||
|
<span class="n">leftover</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="n">start</span><span class="p">,</span> <span class="n">end</span> <span class="o">=</span> <span class="n">__get_results_limits</span><span class="p">(</span><span class="n">pageno</span><span class="p">)</span>
|
||||||
|
<span class="k">with</span> <span class="n">Popen</span><span class="p">(</span><span class="n">cmd</span><span class="p">,</span> <span class="n">stdout</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">env</span><span class="o">=</span><span class="n">environment_variables</span><span class="p">)</span> <span class="k">as</span> <span class="n">process</span><span class="p">:</span>
|
||||||
|
<span class="n">line</span> <span class="o">=</span> <span class="n">process</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
|
||||||
|
<span class="k">while</span> <span class="n">line</span><span class="p">:</span>
|
||||||
|
<span class="n">buf</span> <span class="o">=</span> <span class="n">leftover</span> <span class="o">+</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
|
||||||
|
<span class="n">raw_results</span> <span class="o">=</span> <span class="n">buf</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">result_separator</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">raw_results</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]:</span>
|
||||||
|
<span class="n">leftover</span> <span class="o">=</span> <span class="n">raw_results</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">raw_results</span> <span class="o">=</span> <span class="n">raw_results</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">raw_result</span> <span class="ow">in</span> <span class="n">raw_results</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span> <span class="o">=</span> <span class="n">__parse_single_result</span><span class="p">(</span><span class="n">raw_result</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">_command_logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'skipped result:'</span><span class="p">,</span> <span class="n">raw_result</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">start</span> <span class="o"><=</span> <span class="n">count</span> <span class="ow">and</span> <span class="n">count</span> <span class="o"><=</span> <span class="n">end</span><span class="p">:</span> <span class="c1"># pylint: disable=chained-comparison</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">KeyValue</span><span class="p">(</span><span class="n">kvmap</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">count</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||||
|
<span class="k">if</span> <span class="n">end</span> <span class="o"><</span> <span class="n">count</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
<span class="n">line</span> <span class="o">=</span> <span class="n">process</span><span class="o">.</span><span class="n">stdout</span><span class="o">.</span><span class="n">readline</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">return_code</span> <span class="o">=</span> <span class="n">process</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">return_code</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">'non-zero return code when running command'</span><span class="p">,</span> <span class="n">cmd</span><span class="p">,</span> <span class="n">return_code</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__get_results_limits</span><span class="p">(</span><span class="n">pageno</span><span class="p">):</span>
|
||||||
|
<span class="n">start</span> <span class="o">=</span> <span class="p">(</span><span class="n">pageno</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span>
|
||||||
|
<span class="n">end</span> <span class="o">=</span> <span class="n">start</span> <span class="o">+</span> <span class="mi">9</span>
|
||||||
|
<span class="k">return</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__check_query_params</span><span class="p">(</span><span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">query_type</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">query_type</span> <span class="o">==</span> <span class="s1">'path'</span><span class="p">:</span>
|
||||||
|
<span class="n">query_path</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">query_path</span> <span class="o">=</span> <span class="n">expanduser</span><span class="p">(</span><span class="n">query_path</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">commonprefix</span><span class="p">([</span><span class="n">realpath</span><span class="p">(</span><span class="n">query_path</span><span class="p">),</span> <span class="n">working_dir</span><span class="p">])</span> <span class="o">!=</span> <span class="n">working_dir</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'requested path is outside of configured working directory'</span><span class="p">)</span>
|
||||||
|
<span class="k">elif</span> <span class="n">query_type</span> <span class="o">==</span> <span class="s1">'enum'</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">query_enum</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">param</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">query_enum</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'submitted query params is not allowed'</span><span class="p">,</span> <span class="n">param</span><span class="p">,</span> <span class="s1">'allowed params:'</span><span class="p">,</span> <span class="n">query_enum</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="check_parsing_options">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/offline/command-line-engines.html#searx.engines.command.check_parsing_options">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">check_parsing_options</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Checks if delimiter based parsing or regex parsing is configured correctly"""</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'delimiter'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span> <span class="ow">and</span> <span class="s1">'parse_regex'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'failed to init settings for parsing lines: missing delimiter or parse_regex'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'delimiter'</span> <span class="ow">in</span> <span class="n">engine_settings</span> <span class="ow">and</span> <span class="s1">'parse_regex'</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'failed to init settings for parsing lines: too many settings'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'delimiter'</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'chars'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">'delimiter'</span><span class="p">]</span> <span class="ow">or</span> <span class="s1">'keys'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">'delimiter'</span><span class="p">]:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__parse_single_result</span><span class="p">(</span><span class="n">raw_result</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parses command line output based on configuration"""</span>
|
||||||
|
|
||||||
|
<span class="n">result</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">delimiter</span><span class="p">:</span>
|
||||||
|
<span class="n">elements</span> <span class="o">=</span> <span class="n">raw_result</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">delimiter</span><span class="p">[</span><span class="s1">'chars'</span><span class="p">],</span> <span class="n">maxsplit</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">delimiter</span><span class="p">[</span><span class="s1">'keys'</span><span class="p">])</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">elements</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">delimiter</span><span class="p">[</span><span class="s1">'keys'</span><span class="p">]):</span>
|
||||||
|
<span class="k">return</span> <span class="p">{}</span>
|
||||||
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">elements</span><span class="p">)):</span> <span class="c1"># pylint: disable=consider-using-enumerate</span>
|
||||||
|
<span class="n">result</span><span class="p">[</span><span class="n">delimiter</span><span class="p">[</span><span class="s1">'keys'</span><span class="p">][</span><span class="n">i</span><span class="p">]]</span> <span class="o">=</span> <span class="n">elements</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">parse_regex</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">result_key</span><span class="p">,</span> <span class="n">regex</span> <span class="ow">in</span> <span class="n">_compiled_parse_regex</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">found</span> <span class="o">=</span> <span class="n">regex</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">raw_result</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">found</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="p">{}</span>
|
||||||
|
<span class="n">result</span><span class="p">[</span><span class="n">result_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">raw_result</span><span class="p">[</span><span class="n">found</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> <span class="p">:</span> <span class="n">found</span><span class="o">.</span><span class="n">end</span><span class="p">()]</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">result</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
272
_modules/searx/engines/core.html
Normal file
@@ -0,0 +1,272 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.core — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.core</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.core</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""CORE_ (COnnecting REpositories) provides a comprehensive bibliographic</span>
|
||||||
|
<span class="sd">database of the world’s scholarly literature, collecting and indexing</span>
|
||||||
|
<span class="sd">research from repositories and journals.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _CORE: https://core.ac.uk/about</span>
|
||||||
|
|
||||||
|
<span class="sd">.. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> The CORE engine requires an :py:obj:`API key <api_key>`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _core engine config:</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following additional settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`api_key`</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: core.ac.uk</span>
|
||||||
|
<span class="sd"> api_key: "..."</span>
|
||||||
|
<span class="sd"> inactive: false</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://core.ac.uk"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q22661180"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"https://api.core.ac.uk/docs/v3"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">api_key</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""For an API key register at https://core.ac.uk/services/api and insert</span>
|
||||||
|
<span class="sd">the API key in the engine :ref:`core engine config`."""</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"science"</span><span class="p">,</span> <span class="s2">"scientific publications"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">nb_per_page</span> <span class="o">=</span> <span class="mi">10</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://api.core.ac.uk/v3/search/works/"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/core.html#searx.engines.core.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the CORE_ engine, checks whether the :py:obj:`api_key`</span>
|
||||||
|
<span class="sd"> is set, otherwise the engine is inactive.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"api_key"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">key</span> <span class="ow">and</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"unset"</span><span class="p">,</span> <span class="s2">"unknown"</span><span class="p">,</span> <span class="s2">"..."</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"CORE's API key is not set or invalid."</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="c1"># API v3 uses different parameters</span>
|
||||||
|
<span class="n">search_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"q"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"offset"</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">nb_per_page</span><span class="p">,</span>
|
||||||
|
<span class="s2">"limit"</span><span class="p">:</span> <span class="n">nb_per_page</span><span class="p">,</span>
|
||||||
|
<span class="s2">"sort"</span><span class="p">:</span> <span class="s2">"relevance"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s2">"?"</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">search_params</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"Authorization"</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"Bearer </span><span class="si">{</span><span class="n">api_key</span><span class="si">}</span><span class="s2">"</span><span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"results"</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="c1"># Get title</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"title"</span><span class="p">):</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="c1"># Get URL - try different options</span>
|
||||||
|
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># Try DOI first</span>
|
||||||
|
<span class="n">doi</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"doi"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">doi</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://doi.org/</span><span class="si">{</span><span class="n">doi</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"doi"</span><span class="p">):</span>
|
||||||
|
<span class="c1"># use the DOI reference</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s2">"https://doi.org/"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"doi"</span><span class="p">])</span>
|
||||||
|
<span class="k">elif</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"id"</span><span class="p">):</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s2">"https://core.ac.uk/works/"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"id"</span><span class="p">])</span>
|
||||||
|
<span class="k">elif</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"downloadUrl"</span><span class="p">):</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"downloadUrl"</span><span class="p">]</span>
|
||||||
|
<span class="k">elif</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"sourceFulltextUrls"</span><span class="p">):</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s2">"sourceFulltextUrls"</span><span class="p">]</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="c1"># Published date</span>
|
||||||
|
<span class="n">published_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">raw_date</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"publishedDate"</span><span class="p">)</span> <span class="ow">or</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"depositedDate"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">raw_date</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">published_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromisoformat</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"publishedDate"</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"Z"</span><span class="p">,</span> <span class="s2">"+00:00"</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">AttributeError</span><span class="p">):</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
|
||||||
|
<span class="c1"># Handle journals</span>
|
||||||
|
<span class="n">journals</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"journals"</span><span class="p">):</span>
|
||||||
|
<span class="n">journals</span> <span class="o">=</span> <span class="p">[</span><span class="n">j</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"title"</span><span class="p">)</span> <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">result</span><span class="p">[</span><span class="s2">"journals"</span><span class="p">]</span> <span class="k">if</span> <span class="n">j</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"title"</span><span class="p">)]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Handle publisher</span>
|
||||||
|
<span class="n">publisher</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"publisher"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"'"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Handle authors</span>
|
||||||
|
<span class="n">authors</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"authors"</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">i</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"name"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">name</span><span class="p">:</span>
|
||||||
|
<span class="n">authors</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"title"</span><span class="p">),</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"fullText"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">""</span><span class="p">,</span>
|
||||||
|
<span class="n">tags</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"fieldOfStudy"</span><span class="p">,</span> <span class="p">[]),</span>
|
||||||
|
<span class="n">publishedDate</span><span class="o">=</span><span class="n">published_date</span><span class="p">,</span>
|
||||||
|
<span class="nb">type</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"documentType"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">""</span><span class="p">,</span>
|
||||||
|
<span class="n">authors</span><span class="o">=</span><span class="n">authors</span><span class="p">,</span>
|
||||||
|
<span class="n">editor</span><span class="o">=</span><span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"contributors"</span><span class="p">,</span> <span class="p">[])),</span>
|
||||||
|
<span class="n">publisher</span><span class="o">=</span><span class="n">publisher</span><span class="p">,</span>
|
||||||
|
<span class="n">journal</span><span class="o">=</span><span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">journals</span><span class="p">),</span>
|
||||||
|
<span class="n">doi</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"doi"</span><span class="p">),</span>
|
||||||
|
<span class="n">pdf_url</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"downloadUrl"</span><span class="p">,</span> <span class="p">{})</span> <span class="ow">or</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"sourceFulltextUrls"</span><span class="p">,</span> <span class="p">{}),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
353
_modules/searx/engines/dailymotion.html
Normal file
@@ -0,0 +1,353 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.dailymotion — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.dailymotion</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.dailymotion</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Dailymotion (Videos)</span>
|
||||||
|
<span class="sd">~~~~~~~~~~~~~~~~~~~~</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _REST GET: https://developers.dailymotion.com/tools/</span>
|
||||||
|
<span class="sd">.. _Global API Parameters: https://developers.dailymotion.com/api/#global-parameters</span>
|
||||||
|
<span class="sd">.. _Video filters API: https://developers.dailymotion.com/api/#video-filters</span>
|
||||||
|
<span class="sd">.. _Fields selection: https://developers.dailymotion.com/api/#fields-selection</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span><span class="p">,</span> <span class="n">raise_for_httperror</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineAPIException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">region_tag</span><span class="p">,</span> <span class="n">language_tag</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.dailymotion.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q769222'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://www.dailymotion.com/developer'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'videos'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">number_of_results</span> <span class="o">=</span> <span class="mi">10</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_delta_dict</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"day"</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span>
|
||||||
|
<span class="s2">"week"</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">7</span><span class="p">),</span>
|
||||||
|
<span class="s2">"month"</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">31</span><span class="p">),</span>
|
||||||
|
<span class="s2">"year"</span><span class="p">:</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">365</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="mi">2</span><span class="p">:</span> <span class="p">{</span><span class="s1">'is_created_for_kids'</span><span class="p">:</span> <span class="s1">'true'</span><span class="p">},</span>
|
||||||
|
<span class="mi">1</span><span class="p">:</span> <span class="p">{</span><span class="s1">'is_created_for_kids'</span><span class="p">:</span> <span class="s1">'true'</span><span class="p">},</span>
|
||||||
|
<span class="mi">0</span><span class="p">:</span> <span class="p">{},</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""True if this video is "Created for Kids" / intends to target an audience</span>
|
||||||
|
<span class="sd">under the age of 16 (``is_created_for_kids`` in `Video filters API`_ )</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">family_filter_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="mi">2</span><span class="p">:</span> <span class="s1">'true'</span><span class="p">,</span>
|
||||||
|
<span class="mi">1</span><span class="p">:</span> <span class="s1">'true'</span><span class="p">,</span>
|
||||||
|
<span class="mi">0</span><span class="p">:</span> <span class="s1">'false'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""By default, the family filter is turned on. Setting this parameter to</span>
|
||||||
|
<span class="sd">``false`` will stop filtering-out explicit content from searches and global</span>
|
||||||
|
<span class="sd">contexts (``family_filter`` in `Global API Parameters`_ ).</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">result_fields</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'allow_embed'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'description'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'created_time'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'duration'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail_360_url'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'id'</span><span class="p">,</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="sd">"""`Fields selection`_, by default, a few fields are returned. To request more</span>
|
||||||
|
<span class="sd">specific fields, the ``fields`` parameter is used with the list of fields</span>
|
||||||
|
<span class="sd">SearXNG needs in the response to build a video result list.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">search_url</span> <span class="o">=</span> <span class="s1">'https://api.dailymotion.com/videos?'</span>
|
||||||
|
<span class="sd">"""URL to retrieve a list of videos.</span>
|
||||||
|
|
||||||
|
<span class="sd">- `REST GET`_</span>
|
||||||
|
<span class="sd">- `Global API Parameters`_</span>
|
||||||
|
<span class="sd">- `Video filters API`_</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">iframe_src</span> <span class="o">=</span> <span class="s2">"https://www.dailymotion.com/embed/video/</span><span class="si">{video_id}</span><span class="s2">"</span>
|
||||||
|
<span class="sd">"""URL template to embed video in SearXNG's result list."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">eng_region</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="s1">'en_US'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="s1">'en'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'search'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'family_filter'</span><span class="p">:</span> <span class="n">family_filter_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">],</span> <span class="s1">'false'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'thumbnail_ratio'</span><span class="p">:</span> <span class="s1">'original'</span><span class="p">,</span> <span class="c1"># original|widescreen|square</span>
|
||||||
|
<span class="c1"># https://developers.dailymotion.com/api/#video-filters</span>
|
||||||
|
<span class="s1">'languages'</span><span class="p">:</span> <span class="n">eng_lang</span><span class="p">,</span>
|
||||||
|
<span class="s1">'page'</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'password_protected'</span><span class="p">:</span> <span class="s1">'false'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'private'</span><span class="p">:</span> <span class="s1">'false'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'sort'</span><span class="p">:</span> <span class="s1">'relevance'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'limit'</span><span class="p">:</span> <span class="n">number_of_results</span><span class="p">,</span>
|
||||||
|
<span class="s1">'fields'</span><span class="p">:</span> <span class="s1">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result_fields</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">args</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">safesearch_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">],</span> <span class="p">{}))</span>
|
||||||
|
|
||||||
|
<span class="c1"># Don't add localization and country arguments if the user does select a</span>
|
||||||
|
<span class="c1"># language (:de, :en, ..)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">))</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="c1"># https://developers.dailymotion.com/api/#global-parameters</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'localization'</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_region</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'country'</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_region</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="c1"># Insufficient rights for the `ams_country' parameter of route `GET /videos'</span>
|
||||||
|
<span class="c1"># 'ams_country': eng_region.split('_')[1],</span>
|
||||||
|
|
||||||
|
<span class="n">time_delta</span> <span class="o">=</span> <span class="n">time_delta_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"time_range"</span><span class="p">])</span>
|
||||||
|
<span class="k">if</span> <span class="n">time_delta</span><span class="p">:</span>
|
||||||
|
<span class="n">created_after</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">time_delta</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'created_after'</span><span class="p">]</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timestamp</span><span class="p">(</span><span class="n">created_after</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">query_str</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span> <span class="o">+</span> <span class="n">query_str</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># get response from search-request</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">search_res</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># check for an API error</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'error'</span> <span class="ow">in</span> <span class="n">search_res</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="n">search_res</span><span class="p">[</span><span class="s1">'error'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'message'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse results</span>
|
||||||
|
<span class="k">for</span> <span class="n">res</span> <span class="ow">in</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'list'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">res</span><span class="p">[</span><span class="s1">'description'</span><span class="p">])</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">></span> <span class="mi">300</span><span class="p">:</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="p">[:</span><span class="mi">300</span><span class="p">]</span> <span class="o">+</span> <span class="s1">'...'</span>
|
||||||
|
|
||||||
|
<span class="n">publishedDate</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">res</span><span class="p">[</span><span class="s1">'created_time'</span><span class="p">],</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">length</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">gmtime</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'duration'</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">length</span><span class="o">.</span><span class="n">tm_hour</span><span class="p">:</span>
|
||||||
|
<span class="n">length</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%H:%M:%S"</span><span class="p">,</span> <span class="n">length</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">length</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%M:%S"</span><span class="p">,</span> <span class="n">length</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="s1">'thumbnail_360_url'</span><span class="p">]</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"http://"</span><span class="p">,</span> <span class="s2">"https://"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'videos.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">publishedDate</span><span class="p">,</span>
|
||||||
|
<span class="s1">'length'</span><span class="p">:</span> <span class="n">length</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># HINT: no mater what the value is, without API token videos can't shown</span>
|
||||||
|
<span class="c1"># embedded</span>
|
||||||
|
<span class="k">if</span> <span class="n">res</span><span class="p">[</span><span class="s1">'allow_embed'</span><span class="p">]:</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s1">'iframe_src'</span><span class="p">]</span> <span class="o">=</span> <span class="n">iframe_src</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">video_id</span><span class="o">=</span><span class="n">res</span><span class="p">[</span><span class="s1">'id'</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># return results</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/dailymotion.html#searx.engines.dailymotion.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch locales & languages from dailymotion.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Locales fetched from `api/locales <https://api.dailymotion.com/locales>`_.</span>
|
||||||
|
<span class="sd"> There are duplications in the locale codes returned from Dailymotion which</span>
|
||||||
|
<span class="sd"> can be ignored::</span>
|
||||||
|
|
||||||
|
<span class="sd"> en_EN --> en_GB, en_US</span>
|
||||||
|
<span class="sd"> ar_AA --> ar_EG, ar_AE, ar_SA</span>
|
||||||
|
|
||||||
|
<span class="sd"> The language list `api/languages <https://api.dailymotion.com/languages>`_</span>
|
||||||
|
<span class="sd"> contains over 7000 *languages* codes (see PR1071_). We use only those</span>
|
||||||
|
<span class="sd"> language codes that are used in the locales.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _PR1071: https://github.com/searxng/searxng/pull/1071</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">'https://api.dailymotion.com/locales'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from dailymotion/locales is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="s1">'list'</span><span class="p">]:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'locale'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'en_EN'</span><span class="p">,</span> <span class="s1">'ar_AA'</span><span class="p">):</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: item unknown --> </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
|
||||||
|
<span class="n">locale_lang_list</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">values</span><span class="p">()]</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">'https://api.dailymotion.com/languages'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from dailymotion/languages is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="s1">'list'</span><span class="p">]:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'code'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="n">locale_lang_list</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
233
_modules/searx/engines/demo_offline.html
Normal file
@@ -0,0 +1,233 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.demo_offline — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.demo_offline</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.demo_offline</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Within this module we implement a *demo offline engine*. Do not look to</span>
|
||||||
|
<span class="sd">close to the implementation, its just a simple example.</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">To get in use of this *demo* engine add the following entry to your engines list</span>
|
||||||
|
<span class="sd">in ``settings.yml``:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: my offline engine</span>
|
||||||
|
<span class="sd"> engine: demo_offline</span>
|
||||||
|
<span class="sd"> shortcut: demo</span>
|
||||||
|
<span class="sd"> disabled: false</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">RequestParams</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">engine_type</span> <span class="o">=</span> <span class="s2">"offline"</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"general"</span><span class="p">]</span>
|
||||||
|
<span class="n">disabled</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">timeout</span> <span class="o">=</span> <span class="mf">2.0</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># if there is a need for globals, use a leading underline</span>
|
||||||
|
<span class="n">_my_offline_engine</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="n">CACHE</span><span class="p">:</span> <span class="n">EngineCache</span>
|
||||||
|
<span class="sd">"""Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
|
||||||
|
<span class="sd">seconds."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/demo/demo_offline.html#searx.engines.demo_offline.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Dynamic setup of the engine settings.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The origin of this demo engine is a simple json string which is loaded in</span>
|
||||||
|
<span class="sd"> this example while the engine is initialized.</span>
|
||||||
|
|
||||||
|
<span class="sd"> For more details see :py:obj:`searx.enginelib.Engine.setup`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">global</span> <span class="n">_my_offline_engine</span><span class="p">,</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
|
||||||
|
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">[</span><span class="s2">"name"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="n">_my_offline_engine</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s1">'[ {"value": "</span><span class="si">%s</span><span class="s1">"}'</span>
|
||||||
|
<span class="s1">', {"value":"first item"}'</span>
|
||||||
|
<span class="s1">', {"value":"second item"}'</span>
|
||||||
|
<span class="s1">', {"value":"third item"}'</span>
|
||||||
|
<span class="s1">']'</span> <span class="o">%</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'name'</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="init">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/demo/demo_offline.html#searx.engines.demo_offline.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> For more details see :py:obj:`searx.enginelib.Engine.init`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/demo/demo_offline.html#searx.engines.demo_offline.search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">search</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"RequestParams"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Query (offline) engine and return results. Assemble the list of results</span>
|
||||||
|
<span class="sd"> from your local engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> In this demo engine we ignore the 'query' term, usual you would pass the</span>
|
||||||
|
<span class="sd"> 'query' term to your local engine to filter out the results.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"count"</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="n">data_rows</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">_my_offline_engine</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">data_rows</span><span class="p">:</span>
|
||||||
|
<span class="n">count</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||||
|
<span class="n">kvmap</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'language'</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'value'</span><span class="p">:</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">KeyValue</span><span class="p">(</span>
|
||||||
|
<span class="n">caption</span><span class="o">=</span><span class="sa">f</span><span class="s2">"Demo Offline Engine Result #</span><span class="si">{</span><span class="n">count</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="n">key_title</span><span class="o">=</span><span class="s2">"Name"</span><span class="p">,</span>
|
||||||
|
<span class="n">value_title</span><span class="o">=</span><span class="s2">"Value"</span><span class="p">,</span>
|
||||||
|
<span class="n">kvmap</span><span class="o">=</span><span class="n">kvmap</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="n">number_of_results</span><span class="o">=</span><span class="n">count</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># cache counter value for 20sec</span>
|
||||||
|
<span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">"count"</span><span class="p">,</span> <span class="n">count</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
242
_modules/searx/engines/demo_online.html
Normal file
@@ -0,0 +1,242 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.demo_online — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.demo_online</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.demo_online</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Within this module we implement a *demo online engine*. Do not look to</span>
|
||||||
|
<span class="sd">close to the implementation, its just a simple example which queries `The Art</span>
|
||||||
|
<span class="sd">Institute of Chicago <https://www.artic.edu>`_</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">To get in use of this *demo* engine add the following entry to your engines</span>
|
||||||
|
<span class="sd">list in ``settings.yml``:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: my online engine</span>
|
||||||
|
<span class="sd"> engine: demo_online</span>
|
||||||
|
<span class="sd"> shortcut: demo</span>
|
||||||
|
<span class="sd"> disabled: false</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">engine_type</span> <span class="o">=</span> <span class="s2">"online"</span>
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"general"</span><span class="p">]</span>
|
||||||
|
<span class="n">disabled</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">timeout</span> <span class="o">=</span> <span class="mf">2.0</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"images"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">20</span>
|
||||||
|
|
||||||
|
<span class="n">search_api</span> <span class="o">=</span> <span class="s2">"https://api.artic.edu/api/v1/artworks/search"</span>
|
||||||
|
<span class="n">image_api</span> <span class="o">=</span> <span class="s2">"https://www.artic.edu/iiif/2/"</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://www.artic.edu"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q239303"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"http://api.artic.edu/docs/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># if there is a need for globals, use a leading underline</span>
|
||||||
|
<span class="n">_my_online_engine</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/demo/demo_online.html#searx.engines.demo_online.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Dynamic setup of the engine settings.</span>
|
||||||
|
|
||||||
|
<span class="sd"> For more details see :py:obj:`searx.enginelib.Engine.setup`."""</span>
|
||||||
|
<span class="k">global</span> <span class="n">_my_online_engine</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
<span class="n">_my_online_engine</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"name"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="init">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/demo/demo_online.html#searx.engines.demo_online.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> For more details see :py:obj:`searx.enginelib.Engine.init`."""</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/demo/demo_online.html#searx.engines.demo_online.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Build up the ``params`` for the online request. In this example we build a</span>
|
||||||
|
<span class="sd"> URL to fetch images from `artic.edu <https://artic.edu>`__."""</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s2">"q"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"page"</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">],</span>
|
||||||
|
<span class="s2">"fields"</span><span class="p">:</span> <span class="s2">"id,title,artist_display,medium_display,image_id,date_display,dimensions,artist_titles"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"limit"</span><span class="p">:</span> <span class="n">page_size</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">search_api</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">args</span><span class="si">}</span><span class="s2">"</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/demo/demo_online.html#searx.engines.demo_online.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse out the result items from the response. In this example we parse the</span>
|
||||||
|
<span class="sd"> response from `api.artic.edu <https://artic.edu>`__ and filter out all</span>
|
||||||
|
<span class="sd"> images.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span>
|
||||||
|
<span class="n">answer</span><span class="o">=</span><span class="s2">"this is a dummy answer .."</span><span class="p">,</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="s2">"https://example.org"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">"data"</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">result</span><span class="p">[</span><span class="s2">"image_id"</span><span class="p">]:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">kwargs</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"url"</span><span class="p">:</span> <span class="s2">"https://artic.edu/artworks/</span><span class="si">%(id)s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
|
||||||
|
<span class="s2">"title"</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s2">"title"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">" (</span><span class="si">%(date_display)s</span><span class="s2">) // </span><span class="si">%(artist_display)s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
|
||||||
|
<span class="s2">"content"</span><span class="p">:</span> <span class="s2">"</span><span class="si">%(medium_display)s</span><span class="s2"> // </span><span class="si">%(dimensions)s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
|
||||||
|
<span class="s2">"author"</span><span class="p">:</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"artist_titles"</span><span class="p">]),</span>
|
||||||
|
<span class="s2">"img_src"</span><span class="p">:</span> <span class="n">image_api</span> <span class="o">+</span> <span class="s2">"/</span><span class="si">%(image_id)s</span><span class="s2">/full/843,/0/default.jpg"</span> <span class="o">%</span> <span class="n">result</span><span class="p">,</span>
|
||||||
|
<span class="s2">"template"</span><span class="p">:</span> <span class="s2">"images.html"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
606
_modules/searx/engines/duckduckgo.html
Normal file
@@ -0,0 +1,606 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.duckduckgo — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.duckduckgo</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.duckduckgo</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">DuckDuckGo WEB</span>
|
||||||
|
<span class="sd">~~~~~~~~~~~~~~</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote_plus</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">locales</span><span class="p">,</span>
|
||||||
|
<span class="n">external_bang</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||||
|
<span class="n">extr</span><span class="p">,</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://lite.duckduckgo.com/lite/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q12805'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="sd">"""DuckDuckGo-Lite tries to guess user's preferred language from the HTTP</span>
|
||||||
|
<span class="sd">``Accept-Language``. Optional the user can select a region filter (but not a</span>
|
||||||
|
<span class="sd">language).</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span> <span class="c1"># user can't select but the results are filtered</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s2">"https://html.duckduckgo.com/html/"</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'day'</span><span class="p">:</span> <span class="s1">'d'</span><span class="p">,</span> <span class="s1">'week'</span><span class="p">:</span> <span class="s1">'w'</span><span class="p">,</span> <span class="s1">'month'</span><span class="p">:</span> <span class="s1">'m'</span><span class="p">,</span> <span class="s1">'year'</span><span class="p">:</span> <span class="s1">'y'</span><span class="p">}</span>
|
||||||
|
<span class="n">form_data</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'v'</span><span class="p">:</span> <span class="s1">'l'</span><span class="p">,</span> <span class="s1">'api'</span><span class="p">:</span> <span class="s1">'d.js'</span><span class="p">,</span> <span class="s1">'o'</span><span class="p">:</span> <span class="s1">'json'</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">_CACHE</span><span class="p">:</span> <span class="n">EngineCache</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="sd">"""Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
|
||||||
|
<span class="sd">seconds."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_cache</span><span class="p">():</span>
|
||||||
|
<span class="k">global</span> <span class="n">_CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
<span class="k">if</span> <span class="n">_CACHE</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">_CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="s2">"duckduckgo"</span><span class="p">)</span> <span class="c1"># type:ignore</span>
|
||||||
|
<span class="k">return</span> <span class="n">_CACHE</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_vqd">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo.get_vqd">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_vqd</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">region</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">force_request</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns the ``vqd`` that fits to the *query*.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param query: The query term</span>
|
||||||
|
<span class="sd"> :param region: DDG's region code</span>
|
||||||
|
<span class="sd"> :param force_request: force a request to get a vqd value from DDG</span>
|
||||||
|
|
||||||
|
<span class="sd"> TL;DR; the ``vqd`` value is needed to pass DDG's bot protection and is used</span>
|
||||||
|
<span class="sd"> by all request to DDG:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - DuckDuckGo Lite: ``https://lite.duckduckgo.com/lite`` (POST form data)</span>
|
||||||
|
<span class="sd"> - DuckDuckGo Web: ``https://links.duckduckgo.com/d.js?q=...&vqd=...``</span>
|
||||||
|
<span class="sd"> - DuckDuckGo Images: ``https://duckduckgo.com/i.js??q=...&vqd=...``</span>
|
||||||
|
<span class="sd"> - DuckDuckGo Videos: ``https://duckduckgo.com/v.js??q=...&vqd=...``</span>
|
||||||
|
<span class="sd"> - DuckDuckGo News: ``https://duckduckgo.com/news.js??q=...&vqd=...``</span>
|
||||||
|
|
||||||
|
<span class="sd"> DDG's bot detection is sensitive to the ``vqd`` value. For some search terms</span>
|
||||||
|
<span class="sd"> (such as extremely long search terms that are often sent by bots), no ``vqd``</span>
|
||||||
|
<span class="sd"> value can be determined.</span>
|
||||||
|
|
||||||
|
<span class="sd"> If SearXNG cannot determine a ``vqd`` value, then no request should go out</span>
|
||||||
|
<span class="sd"> to DDG.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. attention::</span>
|
||||||
|
|
||||||
|
<span class="sd"> A request with a wrong ``vqd`` value leads to DDG temporarily putting</span>
|
||||||
|
<span class="sd"> SearXNG's IP on a block list.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Requests from IPs in this block list run into timeouts. Not sure, but it</span>
|
||||||
|
<span class="sd"> seems the block list is a sliding window: to get my IP rid from the bot list</span>
|
||||||
|
<span class="sd"> I had to cool down my IP for 1h (send no requests from that IP to DDG).</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">cache</span> <span class="o">=</span> <span class="n">get_cache</span><span class="p">()</span>
|
||||||
|
<span class="n">key</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">secret_hash</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">//</span><span class="si">{</span><span class="n">region</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">force_request</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"vqd: re-use cached value: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">value</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"vqd: request value from from duckduckgo.com"</span><span class="p">)</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s1">'https://duckduckgo.com/?q=</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">query</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s1">'vqd="'</span><span class="p">,</span> <span class="s1">'"'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"vqd value from duckduckgo.com request: '</span><span class="si">%s</span><span class="s2">'"</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"vqd: can't parse value from ddg response (return empty string)"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"vqd: got HTTP </span><span class="si">%s</span><span class="s2"> from duckduckgo.com"</span><span class="p">,</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
|
||||||
|
<span class="n">cache</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"none vqd value from duckduckgo.com: HTTP </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">value</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set_vqd</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">region</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">cache</span> <span class="o">=</span> <span class="n">get_cache</span><span class="p">()</span>
|
||||||
|
<span class="n">key</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">secret_hash</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">//</span><span class="si">{</span><span class="n">region</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">cache</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="mi">3600</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_ddg_lang">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo.get_ddg_lang">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_ddg_lang</span><span class="p">(</span><span class="n">eng_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">'en_US'</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get DuckDuckGo's language identifier from SearXNG's locale.</span>
|
||||||
|
|
||||||
|
<span class="sd"> DuckDuckGo defines its languages by region codes (see</span>
|
||||||
|
<span class="sd"> :py:obj:`fetch_traits`).</span>
|
||||||
|
|
||||||
|
<span class="sd"> To get region and language of a DDG service use:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)</span>
|
||||||
|
<span class="sd"> eng_lang = get_ddg_lang(traits, params['searxng_locale'])</span>
|
||||||
|
|
||||||
|
<span class="sd"> It might confuse, but the ``l`` value of the cookie is what SearXNG calls</span>
|
||||||
|
<span class="sd"> the *region*:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> # !ddi paris :es-AR --> {'ad': 'es_AR', 'ah': 'ar-es', 'l': 'ar-es'}</span>
|
||||||
|
<span class="sd"> params['cookies']['ad'] = eng_lang</span>
|
||||||
|
<span class="sd"> params['cookies']['ah'] = eng_region</span>
|
||||||
|
<span class="sd"> params['cookies']['l'] = eng_region</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> `DDG-lite <https://lite.duckduckgo.com/lite>`__ and the *no Javascript*</span>
|
||||||
|
<span class="sd"> page https://html.duckduckgo.com/html do not offer a language selection</span>
|
||||||
|
<span class="sd"> to the user, only a region can be selected by the user (``eng_region``</span>
|
||||||
|
<span class="sd"> from the example above). DDG-lite and *no Javascript* store the selected</span>
|
||||||
|
<span class="sd"> region in a cookie::</span>
|
||||||
|
|
||||||
|
<span class="sd"> params['cookies']['kl'] = eng_region # 'ar-es'</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'lang_region'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">sxng_locale</span><span class="p">,</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">ddg_reg_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'tw-tzh'</span><span class="p">:</span> <span class="s1">'zh_TW'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'hk-tzh'</span><span class="p">:</span> <span class="s1">'zh_HK'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ct-ca'</span><span class="p">:</span> <span class="s1">'skip'</span><span class="p">,</span> <span class="c1"># ct-ca and es-ca both map to ca_ES</span>
|
||||||
|
<span class="s1">'es-ca'</span><span class="p">:</span> <span class="s1">'ca_ES'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'id-en'</span><span class="p">:</span> <span class="s1">'id_ID'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'no-no'</span><span class="p">:</span> <span class="s1">'nb_NO'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'jp-jp'</span><span class="p">:</span> <span class="s1">'ja_JP'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'kr-kr'</span><span class="p">:</span> <span class="s1">'ko_KR'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'xa-ar'</span><span class="p">:</span> <span class="s1">'ar_SA'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'sl-sl'</span><span class="p">:</span> <span class="s1">'sl_SI'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'th-en'</span><span class="p">:</span> <span class="s1">'th_TH'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'vn-en'</span><span class="p">:</span> <span class="s1">'vi_VN'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">ddg_lang_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="c1"># use ar --> ar_EG (Egypt's arabic)</span>
|
||||||
|
<span class="s2">"ar_DZ"</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"ar_JO"</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"ar_SA"</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># use bn --> bn_BD</span>
|
||||||
|
<span class="s1">'bn_IN'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># use de --> de_DE</span>
|
||||||
|
<span class="s1">'de_CH'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># use en --> en_US,</span>
|
||||||
|
<span class="s1">'en_AU'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'en_CA'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'en_GB'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># Esperanto</span>
|
||||||
|
<span class="s1">'eo_XX'</span><span class="p">:</span> <span class="s1">'eo'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># use es --> es_ES,</span>
|
||||||
|
<span class="s1">'es_AR'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_CL'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_CO'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_CR'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_EC'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_MX'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_PE'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_UY'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_VE'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># use fr --> rf_FR</span>
|
||||||
|
<span class="s1">'fr_CA'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'fr_CH'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'fr_BE'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># use nl --> nl_NL</span>
|
||||||
|
<span class="s1">'nl_BE'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># use pt --> pt_PT</span>
|
||||||
|
<span class="s1">'pt_BR'</span><span class="p">:</span> <span class="s1">'lang_region'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># skip these languages</span>
|
||||||
|
<span class="s1">'od_IN'</span><span class="p">:</span> <span class="s1">'skip'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'io_XX'</span><span class="p">:</span> <span class="s1">'skip'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'tokipona_XX'</span><span class="p">:</span> <span class="s1">'skip'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">quote_ddg_bangs</span><span class="p">(</span><span class="n">query</span><span class="p">):</span>
|
||||||
|
<span class="c1"># quote ddg bangs</span>
|
||||||
|
<span class="n">query_parts</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="c1"># for val in re.split(r'(\s+)', query):</span>
|
||||||
|
<span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s1">'(\s+)'</span><span class="p">,</span> <span class="n">query</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">val</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'!'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">external_bang</span><span class="o">.</span><span class="n">get_node</span><span class="p">(</span><span class="n">external_bang</span><span class="o">.</span><span class="n">EXTERNAL_BANGS</span><span class="p">,</span> <span class="n">val</span><span class="p">[</span><span class="mi">1</span><span class="p">:]):</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"'</span><span class="si">{</span><span class="n">val</span><span class="si">}</span><span class="s2">'"</span>
|
||||||
|
<span class="n">query_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">val</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">query_parts</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">quote_ddg_bangs</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">query</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">500</span><span class="p">:</span>
|
||||||
|
<span class="c1"># DDG does not accept queries with more than 499 chars</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="n">eng_region</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="c1"># Note: The API is reverse-engineered from DuckDuckGo's HTML webpage</span>
|
||||||
|
<span class="c1"># (https://html.duckduckgo.com/html/) and may be subject to additional bot detection mechanisms</span>
|
||||||
|
<span class="c1"># and breaking changes in the future.</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># The params['data'] dictionary can have the following key parameters, in this order:</span>
|
||||||
|
<span class="c1"># - q (str): Search query string</span>
|
||||||
|
<span class="c1"># - b (str): Beginning parameter - empty string for first page requests</span>
|
||||||
|
<span class="c1"># - s (int): Search offset for pagination</span>
|
||||||
|
<span class="c1"># - nextParams (str): Continuation parameters from previous page response, typically empty</span>
|
||||||
|
<span class="c1"># - v (str): Typically 'l' for subsequent pages</span>
|
||||||
|
<span class="c1"># - o (str): Output format, typically 'json'</span>
|
||||||
|
<span class="c1"># - dc (int): Display count - value equal to offset (s) + 1</span>
|
||||||
|
<span class="c1"># - api (str): API endpoint identifier, typically 'd.js'</span>
|
||||||
|
<span class="c1"># - vqd (str): Validation query digest</span>
|
||||||
|
<span class="c1"># - kl (str): Keyboard language/region code (e.g., 'en-us')</span>
|
||||||
|
<span class="c1"># - df (str): Time filter, maps to values like 'd' (day), 'w' (week), 'm' (month), 'y' (year)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'q'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'b'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="k">elif</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">>=</span> <span class="mi">2</span><span class="p">:</span>
|
||||||
|
<span class="n">offset</span> <span class="o">=</span> <span class="mi">10</span> <span class="o">+</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)</span> <span class="o">*</span> <span class="mi">15</span> <span class="c1"># Page 2 = 10, Page 3+ = 10 + n*15</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'s'</span><span class="p">]</span> <span class="o">=</span> <span class="n">offset</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'nextParams'</span><span class="p">]</span> <span class="o">=</span> <span class="n">form_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'nextParams'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'v'</span><span class="p">]</span> <span class="o">=</span> <span class="n">form_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'v'</span><span class="p">,</span> <span class="s1">'l'</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'o'</span><span class="p">]</span> <span class="o">=</span> <span class="n">form_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'o'</span><span class="p">,</span> <span class="s1">'json'</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'dc'</span><span class="p">]</span> <span class="o">=</span> <span class="n">offset</span> <span class="o">+</span> <span class="mi">1</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'api'</span><span class="p">]</span> <span class="o">=</span> <span class="n">form_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'api'</span><span class="p">,</span> <span class="s1">'d.js'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># vqd is required to request other pages after the first one</span>
|
||||||
|
<span class="n">vqd</span> <span class="o">=</span> <span class="n">get_vqd</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">eng_region</span><span class="p">,</span> <span class="n">force_request</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">vqd</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'vqd'</span><span class="p">]</span> <span class="o">=</span> <span class="n">vqd</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Don't try to call follow up pages without a vqd value.</span>
|
||||||
|
<span class="c1"># DDG recognizes this as a request from a bot. This lowers the</span>
|
||||||
|
<span class="c1"># reputation of the SearXNG IP and DDG starts to activate CAPTCHAs.</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">]</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"zh"</span><span class="p">):</span>
|
||||||
|
<span class="c1"># Some locales (at least China) do not have a "next page" button and DDG</span>
|
||||||
|
<span class="c1"># will return a HTTP/2 403 Forbidden for a request of such a page.</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="c1"># Put empty kl in form data if language/region set to all</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_region</span> <span class="o">==</span> <span class="s2">"wt-wt"</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'kl'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'kl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_region</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'df'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'df'</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'df'</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'kl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_region</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">url</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'method'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'POST'</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Content-Type'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'application/x-www-form-urlencoded'</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Referer'</span><span class="p">]</span> <span class="o">=</span> <span class="n">url</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Sec-Fetch-Dest'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"document"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Sec-Fetch-Mode'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"navigate"</span> <span class="c1"># at least this one is used by ddg's bot detection</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Sec-Fetch-Site'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"same-origin"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Sec-Fetch-User'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"?1"</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"param headers: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"param data: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">])</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"param cookies: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">])</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="is_ddg_captcha">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo.is_ddg_captcha">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_ddg_captcha</span><span class="p">(</span><span class="n">dom</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""In case of CAPTCHA ddg response its own *not a Robot* dialog and is not</span>
|
||||||
|
<span class="sd"> redirected to a CAPTCHA page."""</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//form[@id='challenge-form']"</span><span class="p">))</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">303</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">doc</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">is_ddg_captcha</span><span class="p">(</span><span class="n">doc</span><span class="p">):</span>
|
||||||
|
<span class="c1"># set suspend time to zero is OK --> ddg does not block the IP</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">(</span><span class="n">suspended_time</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">message</span><span class="o">=</span><span class="sa">f</span><span class="s2">"CAPTCHA (</span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'kl'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">form</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="s1">'//input[@name="vqd"]/..'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">form</span><span class="p">):</span>
|
||||||
|
<span class="c1"># some locales (at least China) does not have a "next page" button</span>
|
||||||
|
<span class="n">form</span> <span class="o">=</span> <span class="n">form</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">form_vqd</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">form</span><span class="p">,</span> <span class="s1">'//input[@name="vqd"]/@value'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">set_vqd</span><span class="p">(</span>
|
||||||
|
<span class="n">query</span><span class="o">=</span><span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'q'</span><span class="p">],</span>
|
||||||
|
<span class="n">region</span><span class="o">=</span><span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">][</span><span class="s1">'kl'</span><span class="p">],</span>
|
||||||
|
<span class="n">value</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">form_vqd</span><span class="p">),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># just select "web-result" and ignore results of class "result--ad result--ad--small"</span>
|
||||||
|
<span class="k">for</span> <span class="n">div_result</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="s1">'//div[@id="links"]/div[contains(@class, "web-result")]'</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">div_result</span><span class="p">,</span> <span class="s1">'.//h2/a'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">title</span><span class="p">:</span>
|
||||||
|
<span class="c1"># this is the "No results." item in the result list</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"title"</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">div_result</span><span class="p">,</span> <span class="s1">'.//h2/a/@href'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"content"</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">div_result</span><span class="p">,</span> <span class="s1">'.//a[contains(@class, "result__snippet")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">zero_click_info_xpath</span> <span class="o">=</span> <span class="s1">'//div[@id="zero_click_abstract"]'</span>
|
||||||
|
<span class="n">zero_click</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="n">zero_click_info_xpath</span><span class="p">))</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">zero_click</span> <span class="ow">and</span> <span class="p">(</span>
|
||||||
|
<span class="s2">"Your IP address is"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">zero_click</span>
|
||||||
|
<span class="ow">and</span> <span class="s2">"Your user agent:"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">zero_click</span>
|
||||||
|
<span class="ow">and</span> <span class="s2">"URL Decoded:"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">zero_click</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span>
|
||||||
|
<span class="n">answer</span><span class="o">=</span><span class="n">zero_click</span><span class="p">,</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">doc</span><span class="p">,</span> <span class="s1">'//div[@id="zero_click_abstract"]/a/@href'</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages & regions from DuckDuckGo.</span>
|
||||||
|
|
||||||
|
<span class="sd"> SearXNG's ``all`` locale maps DuckDuckGo's "Alle regions" (``wt-wt``).</span>
|
||||||
|
<span class="sd"> DuckDuckGo's language "Browsers preferred language" (``wt_WT``) makes no</span>
|
||||||
|
<span class="sd"> sense in a SearXNG request since SearXNG's ``all`` will not add a</span>
|
||||||
|
<span class="sd"> ``Accept-Language`` HTTP header. The value in ``engine_traits.all_locale``</span>
|
||||||
|
<span class="sd"> is ``wt-wt`` (the region).</span>
|
||||||
|
|
||||||
|
<span class="sd"> Beside regions DuckDuckGo also defines its languages by region codes. By</span>
|
||||||
|
<span class="sd"> example these are the english languages in DuckDuckGo:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - en_US</span>
|
||||||
|
<span class="sd"> - en_AU</span>
|
||||||
|
<span class="sd"> - en_CA</span>
|
||||||
|
<span class="sd"> - en_GB</span>
|
||||||
|
|
||||||
|
<span class="sd"> The function :py:obj:`get_ddg_lang` evaluates DuckDuckGo's language from</span>
|
||||||
|
<span class="sd"> SearXNG's locale.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches, too-many-statements, disable=import-outside-toplevel</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">js_obj_str_to_python</span>
|
||||||
|
|
||||||
|
<span class="c1"># fetch regions</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s1">'wt-wt'</span>
|
||||||
|
|
||||||
|
<span class="c1"># updated from u661.js to u.7669f071a13a7daa57cb / should be updated automatically?</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">'https://duckduckgo.com/dist/util/u.7669f071a13a7daa57cb.js'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from DuckDuckGo is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">js_code</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s1">'regions:'</span><span class="p">,</span> <span class="s1">',snippetLengths'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="n">regions</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">js_code</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">regions</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">eng_tag</span> <span class="o">==</span> <span class="s1">'wt-wt'</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s1">'wt-wt'</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">region</span> <span class="o">=</span> <span class="n">ddg_reg_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">region</span> <span class="o">==</span> <span class="s1">'skip'</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">region</span><span class="p">:</span>
|
||||||
|
<span class="n">eng_territory</span><span class="p">,</span> <span class="n">eng_lang</span> <span class="o">=</span> <span class="n">eng_tag</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="n">region</span> <span class="o">=</span> <span class="n">eng_lang</span> <span class="o">+</span> <span class="s1">'_'</span> <span class="o">+</span> <span class="n">eng_territory</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">region</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) -> </span><span class="si">%s</span><span class="s2"> is unknown by babel"</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">region</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
|
||||||
|
<span class="c1"># fetch languages</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'lang_region'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="n">js_code</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s1">'languages:'</span><span class="p">,</span> <span class="s1">',regions'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="n">languages</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">js_obj_str_to_python</span><span class="p">(</span><span class="n">js_code</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">eng_lang</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">languages</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">eng_lang</span> <span class="o">==</span> <span class="s1">'wt_WT'</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">babel_tag</span> <span class="o">=</span> <span class="n">ddg_lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">babel_tag</span> <span class="o">==</span> <span class="s1">'skip'</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">babel_tag</span> <span class="o">==</span> <span class="s1">'lang_region'</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">))</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'lang_region'</span><span class="p">][</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">babel_tag</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: language </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) is unknown by babel"</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_lang</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
371
_modules/searx/engines/duckduckgo_definitions.html
Normal file
@@ -0,0 +1,371 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.duckduckgo_definitions — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.duckduckgo_definitions</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.duckduckgo_definitions</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">DuckDuckGo Instant Answer API</span>
|
||||||
|
<span class="sd">~~~~~~~~~~~~~~~~~~~~~~~~~~~~~</span>
|
||||||
|
|
||||||
|
<span class="sd">The `DDG-API <https://duckduckgo.com/api>`__ is no longer documented but from</span>
|
||||||
|
<span class="sd">reverse engineering we can see that some services (e.g. instant answers) still</span>
|
||||||
|
<span class="sd">in use from the DDG search engine.</span>
|
||||||
|
|
||||||
|
<span class="sd">As far we can say the *instant answers* API does not support languages, or at</span>
|
||||||
|
<span class="sd">least we could not find out how language support should work. It seems that</span>
|
||||||
|
<span class="sd">most of the features are based on English terms.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urlparse</span><span class="p">,</span> <span class="n">urljoin</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">WIKIDATA_UNITS</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">html_to_text</span><span class="p">,</span> <span class="n">get_string_replaces_function</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.external_urls</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_external_url</span><span class="p">,</span> <span class="n">get_earth_coordinates_url</span><span class="p">,</span> <span class="n">area_to_osm_zoom</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://duckduckgo.com/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q12805'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://duckduckgo.com/api'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">URL</span> <span class="o">=</span> <span class="s1">'https://api.duckduckgo.com/'</span> <span class="o">+</span> <span class="s1">'?</span><span class="si">{query}</span><span class="s1">&format=json&pretty=0&no_redirect=1&d=1'</span>
|
||||||
|
|
||||||
|
<span class="n">WIKIDATA_PREFIX</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'http://www.wikidata.org/entity/'</span><span class="p">,</span> <span class="s1">'https://www.wikidata.org/entity/'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">replace_http_by_https</span> <span class="o">=</span> <span class="n">get_string_replaces_function</span><span class="p">({</span><span class="s1">'http:'</span><span class="p">:</span> <span class="s1">'https:'</span><span class="p">})</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="is_broken_text">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo_definitions.is_broken_text">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_broken_text</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""duckduckgo may return something like ``<a href="xxxx">http://somewhere Related website<a/>``</span>
|
||||||
|
|
||||||
|
<span class="sd"> The href URL is broken, the "Related website" may contains some HTML.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The best solution seems to ignore these results.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="n">text</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'http'</span><span class="p">)</span> <span class="ow">and</span> <span class="s1">' '</span> <span class="ow">in</span> <span class="n">text</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">result_to_text</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">htmlResult</span><span class="p">):</span>
|
||||||
|
<span class="c1"># TODO : remove result ending with "Meaning" or "Category" # pylint: disable=fixme</span>
|
||||||
|
<span class="n">result</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">htmlResult</span><span class="p">)</span>
|
||||||
|
<span class="n">a</span> <span class="o">=</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//a'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">a</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">a</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span> <span class="o">=</span> <span class="n">text</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_broken_text</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">result</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">URL</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">}))</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-locals, too-many-branches, too-many-statements</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">search_res</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># search_res.get('Entity') possible values (not exhaustive) :</span>
|
||||||
|
<span class="c1"># * continent / country / department / location / waterfall</span>
|
||||||
|
<span class="c1"># * actor / musician / artist</span>
|
||||||
|
<span class="c1"># * book / performing art / film / television / media franchise / concert tour / playwright</span>
|
||||||
|
<span class="c1"># * prepared food</span>
|
||||||
|
<span class="c1"># * website / software / os / programming language / file format / software engineer</span>
|
||||||
|
<span class="c1"># * company</span>
|
||||||
|
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="n">heading</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Heading'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="n">attributes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">urls</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">infobox_id</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">relatedTopics</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="c1"># add answer if there is one</span>
|
||||||
|
<span class="n">answer</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Answer'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">answer</span><span class="p">:</span>
|
||||||
|
<span class="n">answer_type</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AnswerType'</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'AnswerType="</span><span class="si">%s</span><span class="s1">" Answer="</span><span class="si">%s</span><span class="s1">"'</span><span class="p">,</span> <span class="n">answer_type</span><span class="p">,</span> <span class="n">answer</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">answer</span><span class="p">,</span> <span class="nb">str</span><span class="p">)</span> <span class="ow">and</span> <span class="n">answer_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'calc'</span><span class="p">,</span> <span class="s1">'ip'</span><span class="p">]:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span>
|
||||||
|
<span class="n">answer</span><span class="o">=</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">answer</span><span class="p">),</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AbstractURL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># add infobox</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'Definition'</span> <span class="ow">in</span> <span class="n">search_res</span><span class="p">:</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span> <span class="o">+</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Definition'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'Abstract'</span> <span class="ow">in</span> <span class="n">search_res</span><span class="p">:</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span> <span class="o">+</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Abstract'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># image</span>
|
||||||
|
<span class="n">image</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Image'</span><span class="p">)</span>
|
||||||
|
<span class="n">image</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">image</span> <span class="o">==</span> <span class="s1">''</span> <span class="k">else</span> <span class="n">image</span>
|
||||||
|
<span class="k">if</span> <span class="n">image</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">image</span><span class="p">)</span><span class="o">.</span><span class="n">netloc</span> <span class="o">==</span> <span class="s1">''</span><span class="p">:</span>
|
||||||
|
<span class="n">image</span> <span class="o">=</span> <span class="n">urljoin</span><span class="p">(</span><span class="s1">'https://duckduckgo.com'</span><span class="p">,</span> <span class="n">image</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># urls</span>
|
||||||
|
<span class="c1"># Official website, Wikipedia page</span>
|
||||||
|
<span class="k">for</span> <span class="n">ddg_result</span> <span class="ow">in</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Results'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">firstURL</span> <span class="o">=</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'FirstURL'</span><span class="p">)</span>
|
||||||
|
<span class="n">text</span> <span class="o">=</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Text'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">firstURL</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">text</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="n">text</span><span class="p">,</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">firstURL</span><span class="p">})</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="n">heading</span><span class="p">,</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">firstURL</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="c1"># related topics</span>
|
||||||
|
<span class="k">for</span> <span class="n">ddg_result</span> <span class="ow">in</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'RelatedTopics'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'FirstURL'</span> <span class="ow">in</span> <span class="n">ddg_result</span><span class="p">:</span>
|
||||||
|
<span class="n">firstURL</span> <span class="o">=</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'FirstURL'</span><span class="p">)</span>
|
||||||
|
<span class="n">text</span> <span class="o">=</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Text'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_broken_text</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
|
||||||
|
<span class="n">suggestion</span> <span class="o">=</span> <span class="n">result_to_text</span><span class="p">(</span><span class="n">text</span><span class="p">,</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Result'</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">suggestion</span> <span class="o">!=</span> <span class="n">heading</span> <span class="ow">and</span> <span class="n">suggestion</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">suggestion</span><span class="p">})</span>
|
||||||
|
<span class="k">elif</span> <span class="s1">'Topics'</span> <span class="ow">in</span> <span class="n">ddg_result</span><span class="p">:</span>
|
||||||
|
<span class="n">suggestions</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">relatedTopics</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'name'</span><span class="p">:</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Name'</span><span class="p">,</span> <span class="s1">''</span><span class="p">),</span> <span class="s1">'suggestions'</span><span class="p">:</span> <span class="n">suggestions</span><span class="p">})</span>
|
||||||
|
<span class="k">for</span> <span class="n">topic_result</span> <span class="ow">in</span> <span class="n">ddg_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Topics'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">suggestion</span> <span class="o">=</span> <span class="n">result_to_text</span><span class="p">(</span><span class="n">topic_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Text'</span><span class="p">),</span> <span class="n">topic_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Result'</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">suggestion</span> <span class="o">!=</span> <span class="n">heading</span> <span class="ow">and</span> <span class="n">suggestion</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">suggestions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># abstract</span>
|
||||||
|
<span class="n">abstractURL</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AbstractURL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">abstractURL</span> <span class="o">!=</span> <span class="s1">''</span><span class="p">:</span>
|
||||||
|
<span class="c1"># add as result ? problem always in english</span>
|
||||||
|
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">abstractURL</span>
|
||||||
|
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AbstractSource'</span><span class="p">),</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">abstractURL</span><span class="p">,</span> <span class="s1">'official'</span><span class="p">:</span> <span class="kc">True</span><span class="p">})</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">abstractURL</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">heading</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="c1"># definition</span>
|
||||||
|
<span class="n">definitionURL</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DefinitionURL'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">definitionURL</span> <span class="o">!=</span> <span class="s1">''</span><span class="p">:</span>
|
||||||
|
<span class="c1"># add as result ? as answer ? problem always in english</span>
|
||||||
|
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">definitionURL</span>
|
||||||
|
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'DefinitionSource'</span><span class="p">),</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">definitionURL</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="c1"># to merge with wikidata's infobox</span>
|
||||||
|
<span class="k">if</span> <span class="n">infobox_id</span><span class="p">:</span>
|
||||||
|
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">replace_http_by_https</span><span class="p">(</span><span class="n">infobox_id</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># attributes</span>
|
||||||
|
<span class="c1"># some will be converted to urls</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'Infobox'</span> <span class="ow">in</span> <span class="n">search_res</span><span class="p">:</span>
|
||||||
|
<span class="n">infobox</span> <span class="o">=</span> <span class="n">search_res</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Infobox'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'content'</span> <span class="ow">in</span> <span class="n">infobox</span><span class="p">:</span>
|
||||||
|
<span class="n">osm_zoom</span> <span class="o">=</span> <span class="mi">17</span>
|
||||||
|
<span class="n">coordinates</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">for</span> <span class="n">info</span> <span class="ow">in</span> <span class="n">infobox</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'content'</span><span class="p">):</span>
|
||||||
|
<span class="n">data_type</span> <span class="o">=</span> <span class="n">info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'data_type'</span><span class="p">)</span>
|
||||||
|
<span class="n">data_label</span> <span class="o">=</span> <span class="n">info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'label'</span><span class="p">)</span>
|
||||||
|
<span class="n">data_value</span> <span class="o">=</span> <span class="n">info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'value'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Workaround: ddg may return a double quote</span>
|
||||||
|
<span class="k">if</span> <span class="n">data_value</span> <span class="o">==</span> <span class="s1">'""'</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="c1"># Is it an external URL ?</span>
|
||||||
|
<span class="c1"># * imdb_id / facebook_profile / youtube_channel / youtube_video / twitter_profile</span>
|
||||||
|
<span class="c1"># * instagram_profile / rotten_tomatoes / spotify_artist_id / itunes_artist_id / soundcloud_id</span>
|
||||||
|
<span class="c1"># * netflix_id</span>
|
||||||
|
<span class="n">external_url</span> <span class="o">=</span> <span class="n">get_external_url</span><span class="p">(</span><span class="n">data_type</span><span class="p">,</span> <span class="n">data_value</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">external_url</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="n">data_label</span><span class="p">,</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">external_url</span><span class="p">})</span>
|
||||||
|
<span class="k">elif</span> <span class="n">data_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'instance'</span><span class="p">,</span> <span class="s1">'wiki_maps_trigger'</span><span class="p">,</span> <span class="s1">'google_play_artist_id'</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># ignore instance: Wikidata value from "Instance Of" (Qxxxx)</span>
|
||||||
|
<span class="c1"># ignore wiki_maps_trigger: reference to a javascript</span>
|
||||||
|
<span class="c1"># ignore google_play_artist_id: service shutdown</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
<span class="k">elif</span> <span class="n">data_type</span> <span class="o">==</span> <span class="s1">'string'</span> <span class="ow">and</span> <span class="n">data_label</span> <span class="o">==</span> <span class="s1">'Website'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># There is already an URL for the website</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
<span class="k">elif</span> <span class="n">data_type</span> <span class="o">==</span> <span class="s1">'area'</span><span class="p">:</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'label'</span><span class="p">:</span> <span class="n">data_label</span><span class="p">,</span> <span class="s1">'value'</span><span class="p">:</span> <span class="n">area_to_str</span><span class="p">(</span><span class="n">data_value</span><span class="p">),</span> <span class="s1">'entity'</span><span class="p">:</span> <span class="s1">'P2046'</span><span class="p">})</span>
|
||||||
|
<span class="n">osm_zoom</span> <span class="o">=</span> <span class="n">area_to_osm_zoom</span><span class="p">(</span><span class="n">data_value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'amount'</span><span class="p">))</span>
|
||||||
|
<span class="k">elif</span> <span class="n">data_type</span> <span class="o">==</span> <span class="s1">'coordinates'</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">data_value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'globe'</span><span class="p">)</span> <span class="o">==</span> <span class="s1">'http://www.wikidata.org/entity/Q2'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># coordinate on Earth</span>
|
||||||
|
<span class="c1"># get the zoom information from the area</span>
|
||||||
|
<span class="n">coordinates</span> <span class="o">=</span> <span class="n">info</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># coordinate NOT on Earth</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'label'</span><span class="p">:</span> <span class="n">data_label</span><span class="p">,</span> <span class="s1">'value'</span><span class="p">:</span> <span class="n">data_value</span><span class="p">,</span> <span class="s1">'entity'</span><span class="p">:</span> <span class="s1">'P625'</span><span class="p">})</span>
|
||||||
|
<span class="k">elif</span> <span class="n">data_type</span> <span class="o">==</span> <span class="s1">'string'</span><span class="p">:</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'label'</span><span class="p">:</span> <span class="n">data_label</span><span class="p">,</span> <span class="s1">'value'</span><span class="p">:</span> <span class="n">data_value</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">coordinates</span><span class="p">:</span>
|
||||||
|
<span class="n">data_label</span> <span class="o">=</span> <span class="n">coordinates</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'label'</span><span class="p">)</span>
|
||||||
|
<span class="n">data_value</span> <span class="o">=</span> <span class="n">coordinates</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'value'</span><span class="p">)</span>
|
||||||
|
<span class="n">latitude</span> <span class="o">=</span> <span class="n">data_value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'latitude'</span><span class="p">)</span>
|
||||||
|
<span class="n">longitude</span> <span class="o">=</span> <span class="n">data_value</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'longitude'</span><span class="p">)</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">get_earth_coordinates_url</span><span class="p">(</span><span class="n">latitude</span><span class="p">,</span> <span class="n">longitude</span><span class="p">,</span> <span class="n">osm_zoom</span><span class="p">)</span>
|
||||||
|
<span class="n">urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="s1">'OpenStreetMap'</span><span class="p">,</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'entity'</span><span class="p">:</span> <span class="s1">'P625'</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">heading</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="c1"># TODO get infobox.meta.value where .label='article_title' # pylint: disable=fixme</span>
|
||||||
|
<span class="k">if</span> <span class="n">image</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">attributes</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">urls</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">relatedTopics</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">urls</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'url'</span><span class="p">],</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">heading</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">})</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'infobox'</span><span class="p">:</span> <span class="n">heading</span><span class="p">,</span>
|
||||||
|
<span class="s1">'id'</span><span class="p">:</span> <span class="n">infobox_id</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">image</span><span class="p">,</span>
|
||||||
|
<span class="s1">'attributes'</span><span class="p">:</span> <span class="n">attributes</span><span class="p">,</span>
|
||||||
|
<span class="s1">'urls'</span><span class="p">:</span> <span class="n">urls</span><span class="p">,</span>
|
||||||
|
<span class="s1">'relatedTopics'</span><span class="p">:</span> <span class="n">relatedTopics</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">unit_to_str</span><span class="p">(</span><span class="n">unit</span><span class="p">):</span>
|
||||||
|
<span class="k">for</span> <span class="n">prefix</span> <span class="ow">in</span> <span class="n">WIKIDATA_PREFIX</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">unit</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">prefix</span><span class="p">):</span>
|
||||||
|
<span class="n">wikidata_entity</span> <span class="o">=</span> <span class="n">unit</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">prefix</span><span class="p">)</span> <span class="p">:]</span>
|
||||||
|
<span class="n">real_unit</span> <span class="o">=</span> <span class="n">WIKIDATA_UNITS</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">wikidata_entity</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">real_unit</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">unit</span>
|
||||||
|
<span class="k">return</span> <span class="n">real_unit</span><span class="p">[</span><span class="s1">'symbol'</span><span class="p">]</span>
|
||||||
|
<span class="k">return</span> <span class="n">unit</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="area_to_str">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/duckduckgo.html#searx.engines.duckduckgo_definitions.area_to_str">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">area_to_str</span><span class="p">(</span><span class="n">area</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""parse ``{'unit': 'https://www.wikidata.org/entity/Q712226', 'amount': '+20.99'}``"""</span>
|
||||||
|
<span class="n">unit</span> <span class="o">=</span> <span class="n">unit_to_str</span><span class="p">(</span><span class="n">area</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'unit'</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">unit</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">amount</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">area</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'amount'</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'</span><span class="si">{}</span><span class="s1"> </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">amount</span><span class="p">,</span> <span class="n">unit</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'</span><span class="si">{}</span><span class="s1"> </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">area</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'amount'</span><span class="p">,</span> <span class="s1">''</span><span class="p">),</span> <span class="n">area</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'unit'</span><span class="p">,</span> <span class="s1">''</span><span class="p">))</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
360
_modules/searx/engines/github_code.html
Normal file
@@ -0,0 +1,360 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.github_code — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.github_code</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.github_code</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-lat_er</span>
|
||||||
|
<span class="sd">"""GitHub code search with `search syntax`_ as described in `Constructing a</span>
|
||||||
|
<span class="sd">search query`_ in the documentation of GitHub's REST API.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _search syntax:</span>
|
||||||
|
<span class="sd"> https://docs.github.com/en/search-github/getting-started-with-searching-on-github/understanding-the-search-syntax</span>
|
||||||
|
<span class="sd">.. _Constructing a search query:</span>
|
||||||
|
<span class="sd"> https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#constructing-a-search-query</span>
|
||||||
|
<span class="sd">.. _Github REST API for code search:</span>
|
||||||
|
<span class="sd"> https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code</span>
|
||||||
|
<span class="sd">.. _Github REST API auth for code search:</span>
|
||||||
|
<span class="sd"> https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code--fine-grained-access-tokens</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following mandatory setting:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`ghc_auth`</span>
|
||||||
|
<span class="sd"> Change the authentication method used when using the API, defaults to none.</span>
|
||||||
|
|
||||||
|
<span class="sd">Optional settings are:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`ghc_highlight_matching_lines`</span>
|
||||||
|
<span class="sd"> Control the highlighting of the matched text (turns off/on).</span>
|
||||||
|
<span class="sd">- :py:obj:`ghc_strip_new_lines`</span>
|
||||||
|
<span class="sd"> Strip new lines at the start or end of each code fragment.</span>
|
||||||
|
<span class="sd">- :py:obj:`ghc_strip_whitespace`</span>
|
||||||
|
<span class="sd"> Strip any whitespace at the start or end of each code fragment.</span>
|
||||||
|
<span class="sd">- :py:obj:`ghc_insert_block_separator`</span>
|
||||||
|
<span class="sd"> Add a `...` between each code fragment before merging them.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: github code</span>
|
||||||
|
<span class="sd"> engine: github_code</span>
|
||||||
|
<span class="sd"> shortcut: ghc</span>
|
||||||
|
<span class="sd"> ghc_auth:</span>
|
||||||
|
<span class="sd"> type: "none"</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: github code</span>
|
||||||
|
<span class="sd"> engine: github_code</span>
|
||||||
|
<span class="sd"> shortcut: ghc</span>
|
||||||
|
<span class="sd"> ghc_auth:</span>
|
||||||
|
<span class="sd"> type: "personal_access_token"</span>
|
||||||
|
<span class="sd"> token: "<token>"</span>
|
||||||
|
<span class="sd"> ghc_highlight_matching_lines: true</span>
|
||||||
|
<span class="sd"> ghc_strip_whitespace: true</span>
|
||||||
|
<span class="sd"> ghc_strip_new_lines: true</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd"> - name: github code</span>
|
||||||
|
<span class="sd"> engine: github_code</span>
|
||||||
|
<span class="sd"> shortcut: ghc</span>
|
||||||
|
<span class="sd"> ghc_auth:</span>
|
||||||
|
<span class="sd"> type: "bearer"</span>
|
||||||
|
<span class="sd"> token: "<token>"</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementation</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">GitHub does not return the code line indices alongside the code fragment in the</span>
|
||||||
|
<span class="sd">search API. Since these are not super important for the user experience all the</span>
|
||||||
|
<span class="sd">code lines are just relabeled (starting from 1) and appended (a disjoint set of</span>
|
||||||
|
<span class="sd">code blocks in a single file might be returned from the API).</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://github.com/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q364'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'code'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">search_url</span> <span class="o">=</span> <span class="s1">'https://api.github.com/search/code?sort=indexed&</span><span class="si">{query}</span><span class="s1">&</span><span class="si">{page}</span><span class="s1">'</span>
|
||||||
|
<span class="c1"># https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#text-match-metadata</span>
|
||||||
|
<span class="n">accept_header</span> <span class="o">=</span> <span class="s1">'application/vnd.github.text-match+json'</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">ghc_auth</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"none"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"token"</span><span class="p">:</span> <span class="s2">""</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""Change the method of authenticating to the github API.</span>
|
||||||
|
|
||||||
|
<span class="sd">``type`` needs to be one of ``none``, ``personal_access_token``, or ``bearer``.</span>
|
||||||
|
<span class="sd">When type is not `none` a token is expected to be passed as well in</span>
|
||||||
|
<span class="sd">``auth.token``.</span>
|
||||||
|
|
||||||
|
<span class="sd">If there is any privacy concerns about generating a token, one can use the API</span>
|
||||||
|
<span class="sd">without authentication. The calls will be heavily rate limited, this is what the</span>
|
||||||
|
<span class="sd">API returns on such calls::</span>
|
||||||
|
|
||||||
|
<span class="sd"> API rate limit exceeded for <redacted ip>.</span>
|
||||||
|
<span class="sd"> (But here's the good news: Authenticated requests get a higher rate limit)</span>
|
||||||
|
|
||||||
|
<span class="sd">The personal access token or a bearer for an org or a group can be generated [in</span>
|
||||||
|
<span class="sd">the `GitHub settings`_.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _GitHub settings:</span>
|
||||||
|
<span class="sd"> https://docs.github.com/en/rest/search/search?apiVersion=2022-11-28#search-code--fine-grained-access-tokens</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">ghc_highlight_matching_lines</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="sd">"""Highlight the matching code lines."""</span>
|
||||||
|
|
||||||
|
<span class="n">ghc_strip_new_lines</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="sd">"""Strip leading and trailing newlines for each returned fragment.</span>
|
||||||
|
<span class="sd">Single file might return multiple code fragments.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">ghc_strip_whitespace</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">"""Strip all leading and trailing whitespace for each returned fragment.</span>
|
||||||
|
<span class="sd">Single file might return multiple code fragments. Enabling this might break</span>
|
||||||
|
<span class="sd">code indentation.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">ghc_api_version</span> <span class="o">=</span> <span class="s2">"2022-11-28"</span>
|
||||||
|
<span class="sd">"""The version of the GitHub REST API.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">ghc_insert_block_separator</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">"""Each file possibly consists of more than one code block that matches the</span>
|
||||||
|
<span class="sd">search, if this is set to true, the blocks will be separated with ``...`` line.</span>
|
||||||
|
<span class="sd">This might break the lexer and thus result in the lack of code highlighting.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">}),</span> <span class="n">page</span><span class="o">=</span><span class="n">urlencode</span><span class="p">({</span><span class="s1">'page'</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]}))</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Accept'</span><span class="p">]</span> <span class="o">=</span> <span class="n">accept_header</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'X-GitHub-Api-Version'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ghc_api_version</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">ghc_auth</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"none"</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Without the auth header the query fails, so add a dummy instead.</span>
|
||||||
|
<span class="c1"># Queries without auth are heavily rate limited.</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Authorization'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"placeholder"</span>
|
||||||
|
<span class="k">if</span> <span class="n">ghc_auth</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"personal_access_token"</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Authorization'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"token </span><span class="si">{</span><span class="n">ghc_auth</span><span class="p">[</span><span class="s1">'token'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">if</span> <span class="n">ghc_auth</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"bearer"</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Authorization'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Bearer </span><span class="si">{</span><span class="n">ghc_auth</span><span class="p">[</span><span class="s1">'token'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="extract_code">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/github_code.html#searx.engines.github_code.extract_code">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">extract_code</span><span class="p">(</span><span class="n">code_matches</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]])</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">set</span><span class="p">[</span><span class="nb">int</span><span class="p">]]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""</span>
|
||||||
|
<span class="sd"> Iterate over multiple possible matches, for each extract a code fragment.</span>
|
||||||
|
<span class="sd"> Github additionally sends context for _word_ highlights; pygments supports</span>
|
||||||
|
<span class="sd"> highlighting lines, as such we calculate which lines to highlight while</span>
|
||||||
|
<span class="sd"> traversing the text.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">lines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">highlighted_lines_index</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">match</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">code_matches</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">i</span> <span class="o">></span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">ghc_insert_block_separator</span><span class="p">:</span>
|
||||||
|
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"..."</span><span class="p">)</span>
|
||||||
|
<span class="n">buffer</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">highlight_groups</span> <span class="o">=</span> <span class="p">[</span><span class="n">highlight_group</span><span class="p">[</span><span class="s1">'indices'</span><span class="p">]</span> <span class="k">for</span> <span class="n">highlight_group</span> <span class="ow">in</span> <span class="n">match</span><span class="p">[</span><span class="s1">'matches'</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="n">code</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="s1">'fragment'</span><span class="p">]</span>
|
||||||
|
<span class="n">original_code_lenght</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">code</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">ghc_strip_whitespace</span><span class="p">:</span>
|
||||||
|
<span class="n">code</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">ghc_strip_new_lines</span><span class="p">:</span>
|
||||||
|
<span class="n">code</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">offset</span> <span class="o">=</span> <span class="n">original_code_lenght</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">code</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">ghc_strip_whitespace</span><span class="p">:</span>
|
||||||
|
<span class="n">code</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">ghc_strip_new_lines</span><span class="p">:</span>
|
||||||
|
<span class="n">code</span> <span class="o">=</span> <span class="n">code</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">letter</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">code</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">highlight_groups</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="c1"># the API ensures these are sorted already, and we have a</span>
|
||||||
|
<span class="c1"># guaranteed match in the code (all indices are in the range 0</span>
|
||||||
|
<span class="c1"># and len(fragment)), so only check the first highlight group</span>
|
||||||
|
<span class="p">[</span><span class="n">after</span><span class="p">,</span> <span class="n">before</span><span class="p">]</span> <span class="o">=</span> <span class="n">highlight_groups</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">after</span> <span class="o"><=</span> <span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="n">offset</span><span class="p">)</span> <span class="o"><</span> <span class="n">before</span><span class="p">:</span>
|
||||||
|
<span class="c1"># pygments enumerates lines from 1, highlight the next line</span>
|
||||||
|
<span class="n">highlighted_lines_index</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">lines</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">highlight_groups</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">letter</span> <span class="o">==</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">:</span>
|
||||||
|
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">buffer</span><span class="p">))</span>
|
||||||
|
<span class="n">buffer</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">buffer</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">letter</span><span class="p">)</span>
|
||||||
|
<span class="n">lines</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">buffer</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">lines</span><span class="p">,</span> <span class="n">highlighted_lines_index</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">422</span><span class="p">:</span>
|
||||||
|
<span class="c1"># on a invalid search term the status code 422 "Unprocessable Content"</span>
|
||||||
|
<span class="c1"># is returned / e.g. search term is "user: foo" instead "user:foo"</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
<span class="c1"># raise for other errors</span>
|
||||||
|
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'items'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">repo</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'repository'</span><span class="p">]</span> <span class="c1"># pyright: ignore[reportAny]</span>
|
||||||
|
<span class="n">text_matches</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'text_matches'</span><span class="p">]</span> <span class="c1"># pyright: ignore[reportAny]</span>
|
||||||
|
<span class="c1"># ensure picking only the code contents in the blob</span>
|
||||||
|
<span class="n">code_matches</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="n">match</span> <span class="k">for</span> <span class="n">match</span> <span class="ow">in</span> <span class="n">text_matches</span> <span class="k">if</span> <span class="n">match</span><span class="p">[</span><span class="s2">"object_type"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"FileContent"</span> <span class="ow">and</span> <span class="n">match</span><span class="p">[</span><span class="s2">"property"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"content"</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="n">lines</span><span class="p">,</span> <span class="n">highlighted_lines_index</span> <span class="o">=</span> <span class="n">extract_code</span><span class="p">(</span><span class="n">code_matches</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ghc_highlight_matching_lines</span><span class="p">:</span>
|
||||||
|
<span class="n">highlighted_lines_index</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Code</span><span class="p">(</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">item</span><span class="p">[</span><span class="s2">"html_url"</span><span class="p">],</span> <span class="c1"># pyright: ignore[reportAny]</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">repo</span><span class="p">[</span><span class="s1">'full_name'</span><span class="p">]</span><span class="si">}</span><span class="s2"> · </span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="n">filename</span><span class="o">=</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s1">'path'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">repo</span><span class="p">[</span><span class="s1">'description'</span><span class="p">],</span>
|
||||||
|
<span class="n">repository</span><span class="o">=</span><span class="n">repo</span><span class="p">[</span><span class="s1">'html_url'</span><span class="p">],</span>
|
||||||
|
<span class="n">codelines</span><span class="o">=</span><span class="p">[(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span> <span class="k">for</span> <span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="n">line</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">lines</span><span class="p">)],</span>
|
||||||
|
<span class="n">hl_lines</span><span class="o">=</span><span class="n">highlighted_lines_index</span><span class="p">,</span>
|
||||||
|
<span class="n">strip_whitespace</span><span class="o">=</span><span class="n">ghc_strip_whitespace</span><span class="p">,</span>
|
||||||
|
<span class="n">strip_new_lines</span><span class="o">=</span><span class="n">ghc_strip_new_lines</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
640
_modules/searx/engines/google.html
Normal file
@@ -0,0 +1,640 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.google — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.google</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.google</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This is the implementation of the Google WEB engine. Some of this</span>
|
||||||
|
<span class="sd">implementations (manly the :py:obj:`get_google_info`) are shared by other</span>
|
||||||
|
<span class="sd">engines:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :ref:`google images engine`</span>
|
||||||
|
<span class="sd">- :ref:`google news engine`</span>
|
||||||
|
<span class="sd">- :ref:`google videos engine`</span>
|
||||||
|
<span class="sd">- :ref:`google scholar engine`</span>
|
||||||
|
<span class="sd">- :ref:`google autocomplete`</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">string</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.core</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.languages</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">eval_xpath_getindex</span><span class="p">,</span> <span class="n">gen_gsa_useragent</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span><span class="p">,</span> <span class="n">region_tag</span><span class="p">,</span> <span class="n">get_official_locales</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.google.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q9366'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developers.google.com/custom-search/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">50</span>
|
||||||
|
<span class="sd">"""`Google max 50 pages`_</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'day'</span><span class="p">:</span> <span class="s1">'d'</span><span class="p">,</span> <span class="s1">'week'</span><span class="p">:</span> <span class="s1">'w'</span><span class="p">,</span> <span class="s1">'month'</span><span class="p">:</span> <span class="s1">'m'</span><span class="p">,</span> <span class="s1">'year'</span><span class="p">:</span> <span class="s1">'y'</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># Filter results. 0: None, 1: Moderate, 2: Strict</span>
|
||||||
|
<span class="n">filter_mapping</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'off'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'medium'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'high'</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># specific xpath variables</span>
|
||||||
|
<span class="c1"># ------------------------</span>
|
||||||
|
|
||||||
|
<span class="c1"># Suggestions are links placed in a *card-section*, we extract only the text</span>
|
||||||
|
<span class="c1"># from the links not the links itself.</span>
|
||||||
|
<span class="n">suggestion_xpath</span> <span class="o">=</span> <span class="s1">'//div[contains(@class, "ouy7Mc")]//a'</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">_arcid_range</span> <span class="o">=</span> <span class="n">string</span><span class="o">.</span><span class="n">ascii_letters</span> <span class="o">+</span> <span class="n">string</span><span class="o">.</span><span class="n">digits</span> <span class="o">+</span> <span class="s2">"_-"</span>
|
||||||
|
<span class="n">_arcid_random</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ui_async">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.ui_async">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">ui_async</span><span class="p">(</span><span class="n">start</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Format of the response from UI's async request.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``arc_id:<...>,use_ac:true,_fmt:prog``</span>
|
||||||
|
|
||||||
|
<span class="sd"> The arc_id is random generated every hour.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">global</span> <span class="n">_arcid_random</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
|
||||||
|
<span class="n">use_ac</span> <span class="o">=</span> <span class="s2">"use_ac:true"</span>
|
||||||
|
<span class="c1"># _fmt:html returns a HTTP 500 when user search for celebrities like</span>
|
||||||
|
<span class="c1"># '!google natasha allegri' or '!google chris evans'</span>
|
||||||
|
<span class="n">_fmt</span> <span class="o">=</span> <span class="s2">"_fmt:prog"</span>
|
||||||
|
|
||||||
|
<span class="c1"># create a new random arc_id every hour</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">_arcid_random</span> <span class="ow">or</span> <span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">-</span> <span class="n">_arcid_random</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">></span> <span class="mi">3600</span><span class="p">:</span>
|
||||||
|
<span class="n">_arcid_random</span> <span class="o">=</span> <span class="p">(</span><span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">choices</span><span class="p">(</span><span class="n">_arcid_range</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="mi">23</span><span class="p">)),</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()))</span>
|
||||||
|
<span class="n">arc_id</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"arc_id:srp_</span><span class="si">{</span><span class="n">_arcid_random</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">_1</span><span class="si">{</span><span class="n">start</span><span class="si">:</span><span class="s2">02</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">arc_id</span><span class="p">,</span> <span class="n">use_ac</span><span class="p">,</span> <span class="n">_fmt</span><span class="p">])</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_google_info">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.get_google_info">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">,</span> <span class="n">eng_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Composing various (language) properties for the google engines (:ref:`google</span>
|
||||||
|
<span class="sd"> API`).</span>
|
||||||
|
|
||||||
|
<span class="sd"> This function is called by the various google engines (:ref:`google web</span>
|
||||||
|
<span class="sd"> engine`, :ref:`google images engine`, :ref:`google news engine` and</span>
|
||||||
|
<span class="sd"> :ref:`google videos engine`).</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param dict param: Request parameters of the engine. At least</span>
|
||||||
|
<span class="sd"> a ``searxng_locale`` key should be in the dictionary.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param eng_traits: Engine's traits fetched from google preferences</span>
|
||||||
|
<span class="sd"> (:py:obj:`searx.enginelib.traits.EngineTraits`)</span>
|
||||||
|
|
||||||
|
<span class="sd"> :rtype: dict</span>
|
||||||
|
<span class="sd"> :returns:</span>
|
||||||
|
<span class="sd"> Py-Dictionary with the key/value pairs:</span>
|
||||||
|
|
||||||
|
<span class="sd"> language:</span>
|
||||||
|
<span class="sd"> The language code that is used by google (e.g. ``lang_en`` or</span>
|
||||||
|
<span class="sd"> ``lang_zh-TW``)</span>
|
||||||
|
|
||||||
|
<span class="sd"> country:</span>
|
||||||
|
<span class="sd"> The country code that is used by google (e.g. ``US`` or ``TW``)</span>
|
||||||
|
|
||||||
|
<span class="sd"> locale:</span>
|
||||||
|
<span class="sd"> A instance of :py:obj:`babel.core.Locale` build from the</span>
|
||||||
|
<span class="sd"> ``searxng_locale`` value.</span>
|
||||||
|
|
||||||
|
<span class="sd"> subdomain:</span>
|
||||||
|
<span class="sd"> Google subdomain :py:obj:`google_domains` that fits to the country</span>
|
||||||
|
<span class="sd"> code.</span>
|
||||||
|
|
||||||
|
<span class="sd"> params:</span>
|
||||||
|
<span class="sd"> Py-Dictionary with additional request arguments (can be passed to</span>
|
||||||
|
<span class="sd"> :py:func:`urllib.parse.urlencode`).</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``hl`` parameter: specifies the interface language of user interface.</span>
|
||||||
|
<span class="sd"> - ``lr`` parameter: restricts search results to documents written in</span>
|
||||||
|
<span class="sd"> a particular language.</span>
|
||||||
|
<span class="sd"> - ``cr`` parameter: restricts search results to documents</span>
|
||||||
|
<span class="sd"> originating in a particular country.</span>
|
||||||
|
<span class="sd"> - ``ie`` parameter: sets the character encoding scheme that should</span>
|
||||||
|
<span class="sd"> be used to interpret the query string ('utf8').</span>
|
||||||
|
<span class="sd"> - ``oe`` parameter: sets the character encoding scheme that should</span>
|
||||||
|
<span class="sd"> be used to decode the XML result ('utf8').</span>
|
||||||
|
|
||||||
|
<span class="sd"> headers:</span>
|
||||||
|
<span class="sd"> Py-Dictionary with additional HTTP headers (can be passed to</span>
|
||||||
|
<span class="sd"> request's headers)</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``Accept: '*/*``</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">ret_val</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'language'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s1">'country'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s1">'subdomain'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s1">'params'</span><span class="p">:</span> <span class="p">{},</span>
|
||||||
|
<span class="s1">'headers'</span><span class="p">:</span> <span class="p">{},</span>
|
||||||
|
<span class="s1">'cookies'</span><span class="p">:</span> <span class="p">{},</span>
|
||||||
|
<span class="s1">'locale'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">sxng_locale</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'searxng_locale'</span><span class="p">,</span> <span class="s1">'all'</span><span class="p">)</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="s1">'lang_en'</span><span class="p">)</span>
|
||||||
|
<span class="n">lang_code</span> <span class="o">=</span> <span class="n">eng_lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="c1"># lang_zh-TW --> zh-TW / lang_en --> en</span>
|
||||||
|
<span class="n">country</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Test zh_hans & zh_hant --> in the topmost links in the result list of list</span>
|
||||||
|
<span class="c1"># TW and HK you should a find wiktionary.org zh_hant link. In the result</span>
|
||||||
|
<span class="c1"># list of zh-CN should not be no hant link instead you should find</span>
|
||||||
|
<span class="c1"># zh.m.wikipedia.org/zh somewhere in the top.</span>
|
||||||
|
|
||||||
|
<span class="c1"># '!go 日 :zh-TW' --> https://zh.m.wiktionary.org/zh-hant/%E6%97%A5</span>
|
||||||
|
<span class="c1"># '!go 日 :zh-CN' --> https://zh.m.wikipedia.org/zh/%E6%97%A5</span>
|
||||||
|
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'country'</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'locale'</span><span class="p">]</span> <span class="o">=</span> <span class="n">locale</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'supported_domains'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">country</span><span class="o">.</span><span class="n">upper</span><span class="p">(),</span> <span class="s1">'www.google.com'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># hl parameter:</span>
|
||||||
|
<span class="c1"># The hl parameter specifies the interface language (host language) of</span>
|
||||||
|
<span class="c1"># your user interface. To improve the performance and the quality of your</span>
|
||||||
|
<span class="c1"># search results, you are strongly encouraged to set this parameter</span>
|
||||||
|
<span class="c1"># explicitly.</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#hlsp</span>
|
||||||
|
<span class="c1"># The Interface Language:</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results_appendices#interfaceLanguages</span>
|
||||||
|
|
||||||
|
<span class="c1"># https://github.com/searxng/searxng/issues/2515#issuecomment-1607150817</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">lang_code</span><span class="si">}</span><span class="s1">-</span><span class="si">{</span><span class="n">country</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="c1"># lr parameter:</span>
|
||||||
|
<span class="c1"># The lr (language restrict) parameter restricts search results to</span>
|
||||||
|
<span class="c1"># documents written in a particular language.</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#lrsp</span>
|
||||||
|
<span class="c1"># Language Collection Values:</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results_appendices#languageCollections</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># To select 'all' languages an empty 'lr' value is used.</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># Different to other google services, Google Scholar supports to select more</span>
|
||||||
|
<span class="c1"># than one language. The languages are separated by a pipe '|' (logical OR).</span>
|
||||||
|
<span class="c1"># By example: &lr=lang_zh-TW%7Clang_de selects articles written in</span>
|
||||||
|
<span class="c1"># traditional chinese OR german language.</span>
|
||||||
|
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'lr'</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span>
|
||||||
|
<span class="k">if</span> <span class="n">sxng_locale</span> <span class="o">==</span> <span class="s1">'all'</span><span class="p">:</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'lr'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
|
||||||
|
<span class="c1"># cr parameter:</span>
|
||||||
|
<span class="c1"># The cr parameter restricts search results to documents originating in a</span>
|
||||||
|
<span class="c1"># particular country.</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#crsp</span>
|
||||||
|
|
||||||
|
<span class="c1"># specify a region (country) only if a region is given in the selected</span>
|
||||||
|
<span class="c1"># locale --> https://github.com/searxng/searxng/issues/2672</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'cr'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">sxng_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">))</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'cr'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'country'</span> <span class="o">+</span> <span class="n">country</span>
|
||||||
|
|
||||||
|
<span class="c1"># gl parameter: (mandatory by Google News)</span>
|
||||||
|
<span class="c1"># The gl parameter value is a two-letter country code. For WebSearch</span>
|
||||||
|
<span class="c1"># results, the gl parameter boosts search results whose country of origin</span>
|
||||||
|
<span class="c1"># matches the parameter value. See the Country Codes section for a list of</span>
|
||||||
|
<span class="c1"># valid values.</span>
|
||||||
|
<span class="c1"># Specifying a gl parameter value in WebSearch requests should improve the</span>
|
||||||
|
<span class="c1"># relevance of results. This is particularly true for international</span>
|
||||||
|
<span class="c1"># customers and, even more specifically, for customers in English-speaking</span>
|
||||||
|
<span class="c1"># countries other than the United States.</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#glsp</span>
|
||||||
|
|
||||||
|
<span class="c1"># https://github.com/searxng/searxng/issues/2515#issuecomment-1606294635</span>
|
||||||
|
<span class="c1"># ret_val['params']['gl'] = country</span>
|
||||||
|
|
||||||
|
<span class="c1"># ie parameter:</span>
|
||||||
|
<span class="c1"># The ie parameter sets the character encoding scheme that should be used</span>
|
||||||
|
<span class="c1"># to interpret the query string. The default ie value is latin1.</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#iesp</span>
|
||||||
|
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'ie'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'utf8'</span>
|
||||||
|
|
||||||
|
<span class="c1"># oe parameter:</span>
|
||||||
|
<span class="c1"># The oe parameter sets the character encoding scheme that should be used</span>
|
||||||
|
<span class="c1"># to decode the XML result. The default oe value is latin1.</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#oesp</span>
|
||||||
|
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'oe'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'utf8'</span>
|
||||||
|
|
||||||
|
<span class="c1"># num parameter:</span>
|
||||||
|
<span class="c1"># The num parameter identifies the number of search results to return.</span>
|
||||||
|
<span class="c1"># The default num value is 10, and the maximum value is 20. If you request</span>
|
||||||
|
<span class="c1"># more than 20 results, only 20 results will be returned.</span>
|
||||||
|
<span class="c1"># https://developers.google.com/custom-search/docs/xml_results#numsp</span>
|
||||||
|
|
||||||
|
<span class="c1"># HINT: seems to have no effect (tested in google WEB & Images)</span>
|
||||||
|
<span class="c1"># ret_val['params']['num'] = 20</span>
|
||||||
|
|
||||||
|
<span class="c1"># HTTP headers</span>
|
||||||
|
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Accept'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'*/*'</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'User-Agent'</span><span class="p">]</span> <span class="o">=</span> <span class="n">gen_gsa_useragent</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># Cookies</span>
|
||||||
|
|
||||||
|
<span class="c1"># - https://github.com/searxng/searxng/pull/1679#issuecomment-1235432746</span>
|
||||||
|
<span class="c1"># - https://github.com/searxng/searxng/issues/1555</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'CONSENT'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"YES+"</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">ret_val</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="o">.</span><span class="n">host</span> <span class="o">==</span> <span class="s1">'sorry.google.com'</span> <span class="ow">or</span> <span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'/sorry'</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Google search request"""</span>
|
||||||
|
<span class="c1"># pylint: disable=line-too-long</span>
|
||||||
|
<span class="n">start</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span>
|
||||||
|
<span class="n">str_async</span> <span class="o">=</span> <span class="n">ui_async</span><span class="p">(</span><span class="n">start</span><span class="p">)</span>
|
||||||
|
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"ARC_ID: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">str_async</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># https://www.google.de/search?q=corona&hl=de&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium</span>
|
||||||
|
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s1">'https://'</span>
|
||||||
|
<span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span>
|
||||||
|
<span class="o">+</span> <span class="s1">'/search'</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"?"</span>
|
||||||
|
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'filter'</span><span class="p">:</span> <span class="s1">'0'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'start'</span><span class="p">:</span> <span class="n">start</span><span class="p">,</span>
|
||||||
|
<span class="c1"># 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i',</span>
|
||||||
|
<span class="c1"># 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG',</span>
|
||||||
|
<span class="c1"># 'cs' : 1,</span>
|
||||||
|
<span class="c1"># 'sa': 'N',</span>
|
||||||
|
<span class="c1"># 'yv': 3,</span>
|
||||||
|
<span class="c1"># 'prmd': 'vin',</span>
|
||||||
|
<span class="c1"># 'ei': 'GASaY6TxOcy_xc8PtYeY6AE',</span>
|
||||||
|
<span class="c1"># 'sa': 'N',</span>
|
||||||
|
<span class="c1"># 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg'</span>
|
||||||
|
<span class="c1"># formally known as use_mobile_ui</span>
|
||||||
|
<span class="s1">'asearch'</span><span class="p">:</span> <span class="s1">'arc'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'async'</span><span class="p">:</span> <span class="n">str_async</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||||
|
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'tbs'</span><span class="p">:</span> <span class="s1">'qdr:'</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]})</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]:</span>
|
||||||
|
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'safe'</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]})</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA</span>
|
||||||
|
<span class="c1"># ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26;</span>
|
||||||
|
<span class="n">RE_DATA_IMAGE</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);'</span><span class="p">)</span>
|
||||||
|
<span class="n">RE_DATA_IMAGE_end</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*)$'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_data_images</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">data_image_map</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">img_id</span><span class="p">,</span> <span class="n">data_image</span> <span class="ow">in</span> <span class="n">RE_DATA_IMAGE</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
|
||||||
|
<span class="n">end_pos</span> <span class="o">=</span> <span class="n">data_image</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">'='</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">end_pos</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">data_image</span> <span class="o">=</span> <span class="n">data_image</span><span class="p">[:</span> <span class="n">end_pos</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">data_image_map</span><span class="p">[</span><span class="n">img_id</span><span class="p">]</span> <span class="o">=</span> <span class="n">data_image</span>
|
||||||
|
<span class="n">last</span> <span class="o">=</span> <span class="n">RE_DATA_IMAGE_end</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">last</span><span class="p">:</span>
|
||||||
|
<span class="n">data_image_map</span><span class="p">[</span><span class="n">last</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)]</span> <span class="o">=</span> <span class="n">last</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'data:image objects --> </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">list</span><span class="p">(</span><span class="n">data_image_map</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
|
||||||
|
<span class="k">return</span> <span class="n">data_image_map</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get response from google's search request"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches, too-many-statements</span>
|
||||||
|
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
<span class="n">data_image_map</span> <span class="o">=</span> <span class="n">parse_data_images</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># convert the text to dom</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse results</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "MjjYud")]'</span><span class="p">):</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-nested-blocks</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">title_tag</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[contains(@role, "link")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">title_tag</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="c1"># this not one of the common google results *section*</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'ignoring item from the result_xpath list: missing title'</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">title_tag</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">raw_url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//a/@href'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">raw_url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'ignoring item from the result_xpath list: missing url of title "</span><span class="si">%s</span><span class="s1">"'</span><span class="p">,</span> <span class="n">title</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">raw_url</span><span class="p">[</span><span class="mi">7</span><span class="p">:]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'&sa=U'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># remove the google redirector</span>
|
||||||
|
|
||||||
|
<span class="n">content_nodes</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[contains(@data-sncf, "1")]'</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">content_nodes</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">script</span> <span class="ow">in</span> <span class="n">item</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s2">".//script"</span><span class="p">):</span>
|
||||||
|
<span class="n">script</span><span class="o">.</span><span class="n">getparent</span><span class="p">()</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">script</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">content_nodes</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">content</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'ignoring item from the result_xpath list: missing content of title "</span><span class="si">%s</span><span class="s1">"'</span><span class="p">,</span> <span class="n">title</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">content_nodes</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//img/@src'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">thumbnail</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">thumbnail</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'data:image'</span><span class="p">):</span>
|
||||||
|
<span class="n">img_id</span> <span class="o">=</span> <span class="n">content_nodes</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'.//img/@id'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">img_id</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">data_image_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">img_id</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span> <span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">e</span><span class="p">,</span> <span class="n">exc_info</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse suggestion</span>
|
||||||
|
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
|
||||||
|
<span class="c1"># append suggestion</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
|
||||||
|
|
||||||
|
<span class="c1"># return results</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># get supported languages from their site</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">skip_countries</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="c1"># official language of google-country not in google-languages</span>
|
||||||
|
<span class="s1">'AL'</span><span class="p">,</span> <span class="c1"># Albanien (sq)</span>
|
||||||
|
<span class="s1">'AZ'</span><span class="p">,</span> <span class="c1"># Aserbaidschan (az)</span>
|
||||||
|
<span class="s1">'BD'</span><span class="p">,</span> <span class="c1"># Bangladesch (bn)</span>
|
||||||
|
<span class="s1">'BN'</span><span class="p">,</span> <span class="c1"># Brunei Darussalam (ms)</span>
|
||||||
|
<span class="s1">'BT'</span><span class="p">,</span> <span class="c1"># Bhutan (dz)</span>
|
||||||
|
<span class="s1">'ET'</span><span class="p">,</span> <span class="c1"># Äthiopien (am)</span>
|
||||||
|
<span class="s1">'GE'</span><span class="p">,</span> <span class="c1"># Georgien (ka, os)</span>
|
||||||
|
<span class="s1">'GL'</span><span class="p">,</span> <span class="c1"># Grönland (kl)</span>
|
||||||
|
<span class="s1">'KH'</span><span class="p">,</span> <span class="c1"># Kambodscha (km)</span>
|
||||||
|
<span class="s1">'LA'</span><span class="p">,</span> <span class="c1"># Laos (lo)</span>
|
||||||
|
<span class="s1">'LK'</span><span class="p">,</span> <span class="c1"># Sri Lanka (si, ta)</span>
|
||||||
|
<span class="s1">'ME'</span><span class="p">,</span> <span class="c1"># Montenegro (sr)</span>
|
||||||
|
<span class="s1">'MK'</span><span class="p">,</span> <span class="c1"># Nordmazedonien (mk, sq)</span>
|
||||||
|
<span class="s1">'MM'</span><span class="p">,</span> <span class="c1"># Myanmar (my)</span>
|
||||||
|
<span class="s1">'MN'</span><span class="p">,</span> <span class="c1"># Mongolei (mn)</span>
|
||||||
|
<span class="s1">'MV'</span><span class="p">,</span> <span class="c1"># Malediven (dv) // dv_MV is unknown by babel</span>
|
||||||
|
<span class="s1">'MY'</span><span class="p">,</span> <span class="c1"># Malaysia (ms)</span>
|
||||||
|
<span class="s1">'NP'</span><span class="p">,</span> <span class="c1"># Nepal (ne)</span>
|
||||||
|
<span class="s1">'TJ'</span><span class="p">,</span> <span class="c1"># Tadschikistan (tg)</span>
|
||||||
|
<span class="s1">'TM'</span><span class="p">,</span> <span class="c1"># Turkmenistan (tk)</span>
|
||||||
|
<span class="s1">'UZ'</span><span class="p">,</span> <span class="c1"># Usbekistan (uz)</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">,</span> <span class="n">add_domains</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages from Google."""</span>
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel, too-many-branches</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'supported_domains'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">'https://www.google.com/preferences'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"Response from Google's preferences is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'<?xml version="1.0" encoding="UTF-8"?>'</span><span class="p">,</span> <span class="s1">''</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># supported language codes</span>
|
||||||
|
|
||||||
|
<span class="n">lang_map</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'no'</span><span class="p">:</span> <span class="s1">'nb'</span><span class="p">}</span>
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//select[@name='hl']/option"</span><span class="p">):</span>
|
||||||
|
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"INFO: google UI language </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) is unknown by babel"</span> <span class="o">%</span> <span class="p">(</span><span class="n">eng_lang</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"("</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_lang</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_lang</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'lang_'</span> <span class="o">+</span> <span class="n">eng_lang</span>
|
||||||
|
|
||||||
|
<span class="c1"># alias languages</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">'zh'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'lang_zh-CN'</span>
|
||||||
|
|
||||||
|
<span class="c1"># supported region codes</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//select[@name='gl']/option"</span><span class="p">):</span>
|
||||||
|
<span class="n">eng_country</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">eng_country</span> <span class="ow">in</span> <span class="n">skip_countries</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_country</span> <span class="o">==</span> <span class="s1">'ZZ'</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s1">'ZZ'</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">sxng_locales</span> <span class="o">=</span> <span class="n">get_official_locales</span><span class="p">(</span><span class="n">eng_country</span><span class="p">,</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">keys</span><span class="p">(),</span> <span class="n">regional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">sxng_locales</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: can't map from google country </span><span class="si">%s</span><span class="s2"> (</span><span class="si">%s</span><span class="s2">) to a babel region."</span> <span class="o">%</span> <span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'data-name'</span><span class="p">),</span> <span class="n">eng_country</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">sxng_locale</span> <span class="ow">in</span> <span class="n">sxng_locales</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">region_tag</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">eng_country</span>
|
||||||
|
|
||||||
|
<span class="c1"># alias regions</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="s1">'zh-CN'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'HK'</span>
|
||||||
|
|
||||||
|
<span class="c1"># supported domains</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">add_domains</span><span class="p">:</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">'https://www.google.com/supported_domains'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"Response from https://www.google.com/supported_domains is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">domain</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">():</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">domain</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">domain</span> <span class="ow">or</span> <span class="n">domain</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'.google.com'</span><span class="p">,</span>
|
||||||
|
<span class="p">]:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">region</span> <span class="o">=</span> <span class="n">domain</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'supported_domains'</span><span class="p">][</span><span class="n">region</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'www'</span> <span class="o">+</span> <span class="n">domain</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">if</span> <span class="n">region</span> <span class="o">==</span> <span class="s1">'HK'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># There is no google.cn, we use .com.hk for zh-CN</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'supported_domains'</span><span class="p">][</span><span class="s1">'CN'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'www'</span> <span class="o">+</span> <span class="n">domain</span> <span class="c1"># type: ignore</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
236
_modules/searx/engines/google_images.html
Normal file
@@ -0,0 +1,236 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.google_images — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.google_images</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.google_images</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This is the implementation of the Google Images engine using the internal</span>
|
||||||
|
<span class="sd">Google API used by the Google Go Android app.</span>
|
||||||
|
|
||||||
|
<span class="sd">This internal API offer results in</span>
|
||||||
|
|
||||||
|
<span class="sd">- JSON (``_fmt:json``)</span>
|
||||||
|
<span class="sd">- Protobuf_ (``_fmt:pb``)</span>
|
||||||
|
<span class="sd">- Protobuf_ compressed? (``_fmt:pc``)</span>
|
||||||
|
<span class="sd">- HTML (``_fmt:html``)</span>
|
||||||
|
<span class="sd">- Protobuf_ encoded in JSON (``_fmt:jspb``).</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Protobuf: https://en.wikipedia.org/wiki/Protocol_Buffers</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">get_google_info</span><span class="p">,</span>
|
||||||
|
<span class="n">time_range_dict</span><span class="p">,</span>
|
||||||
|
<span class="n">detect_google_sorry</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://images.google.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q521550'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developers.google.com/custom-search'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'images'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">50</span>
|
||||||
|
<span class="sd">"""`Google max 50 pages`_</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">filter_mapping</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'images'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'active'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'active'</span><span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_images.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Google-Image search request"""</span>
|
||||||
|
|
||||||
|
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s1">'https://'</span>
|
||||||
|
<span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span>
|
||||||
|
<span class="o">+</span> <span class="s1">'/search'</span>
|
||||||
|
<span class="o">+</span> <span class="s1">'?'</span>
|
||||||
|
<span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span> <span class="s1">'tbm'</span><span class="p">:</span> <span class="s2">"isch"</span><span class="p">,</span> <span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">],</span> <span class="s1">'asearch'</span><span class="p">:</span> <span class="s1">'isch'</span><span class="p">})</span>
|
||||||
|
<span class="c1"># don't urlencode this because wildly different AND bad results</span>
|
||||||
|
<span class="c1"># pagination uses Zero-based numbering</span>
|
||||||
|
<span class="o">+</span> <span class="sa">f</span><span class="s1">'&async=_fmt:json,p:1,ijn:</span><span class="si">{</span><span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">]</span><span class="w"> </span><span class="o">-</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||||
|
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'tbs'</span><span class="p">:</span> <span class="s1">'qdr:'</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]})</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]:</span>
|
||||||
|
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'safe'</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]})</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span>
|
||||||
|
<span class="c1"># this ua will allow getting ~50 results instead of 10. #1641</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'User-Agent'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s1">'NSTN/3.60.474802233.release Dalvik/2.1.0 (Linux; U; Android 12;'</span> <span class="sa">f</span><span class="s1">' </span><span class="si">{</span><span class="n">google_info</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"country"</span><span class="p">,</span><span class="w"> </span><span class="s2">"US"</span><span class="p">)</span><span class="si">}</span><span class="s1">) gzip'</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_images.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get response from google's search request"""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">json_start</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'{"ischj":'</span><span class="p">)</span>
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">[</span><span class="n">json_start</span><span class="p">:])</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">"ischj"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"metadata"</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">result_item</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">][</span><span class="s2">"referrer_url"</span><span class="p">],</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">][</span><span class="s2">"page_title"</span><span class="p">],</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"text_in_grid"</span><span class="p">][</span><span class="s2">"snippet"</span><span class="p">],</span>
|
||||||
|
<span class="s1">'source'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">][</span><span class="s2">"site_title"</span><span class="p">],</span>
|
||||||
|
<span class="s1">'resolution'</span><span class="p">:</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s2">"original_image"</span><span class="p">][</span><span class="s2">"width"</span><span class="p">]</span><span class="si">}</span><span class="s1"> x </span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s2">"original_image"</span><span class="p">][</span><span class="s2">"height"</span><span class="p">]</span><span class="si">}</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"original_image"</span><span class="p">][</span><span class="s2">"url"</span><span class="p">],</span>
|
||||||
|
<span class="s1">'thumbnail_src'</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="s2">"thumbnail"</span><span class="p">][</span><span class="s2">"url"</span><span class="p">],</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'images.html'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">author</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'iptc'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'creator'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">author</span><span class="p">:</span>
|
||||||
|
<span class="n">result_item</span><span class="p">[</span><span class="s1">'author'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">author</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">copyright_notice</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'iptc'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'copyright_notice'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">copyright_notice</span><span class="p">:</span>
|
||||||
|
<span class="n">result_item</span><span class="p">[</span><span class="s1">'source'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">' | '</span> <span class="o">+</span> <span class="n">copyright_notice</span>
|
||||||
|
|
||||||
|
<span class="n">freshness_date</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"freshness_date"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">freshness_date</span><span class="p">:</span>
|
||||||
|
<span class="n">result_item</span><span class="p">[</span><span class="s1">'source'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">' | '</span> <span class="o">+</span> <span class="n">freshness_date</span>
|
||||||
|
|
||||||
|
<span class="n">file_size</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'gsa'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'file_size'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">file_size</span><span class="p">:</span>
|
||||||
|
<span class="n">result_item</span><span class="p">[</span><span class="s1">'source'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">' (</span><span class="si">%s</span><span class="s1">)'</span> <span class="o">%</span> <span class="n">file_size</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">result_item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
409
_modules/searx/engines/google_news.html
Normal file
@@ -0,0 +1,409 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.google_news — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.google_news</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.google_news</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This is the implementation of the Google News engine.</span>
|
||||||
|
|
||||||
|
<span class="sd">Google News has a different region handling compared to Google WEB.</span>
|
||||||
|
|
||||||
|
<span class="sd">- the ``ceid`` argument has to be set (:py:obj:`ceid_list`)</span>
|
||||||
|
<span class="sd">- the hl_ argument has to be set correctly (and different to Google WEB)</span>
|
||||||
|
<span class="sd">- the gl_ argument is mandatory</span>
|
||||||
|
|
||||||
|
<span class="sd">If one of this argument is not set correctly, the request is redirected to</span>
|
||||||
|
<span class="sd">CONSENT dialog::</span>
|
||||||
|
|
||||||
|
<span class="sd"> https://consent.google.com/m?continue=</span>
|
||||||
|
|
||||||
|
<span class="sd">The google news API ignores some parameters from the common :ref:`google API`:</span>
|
||||||
|
|
||||||
|
<span class="sd">- num_ : the number of search results is ignored / there is no paging all</span>
|
||||||
|
<span class="sd"> results for a query term are in the first response.</span>
|
||||||
|
<span class="sd">- save_ : is ignored / Google-News results are always *SafeSearch*</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _hl: https://developers.google.com/custom-search/docs/xml_results#hlsp</span>
|
||||||
|
<span class="sd">.. _gl: https://developers.google.com/custom-search/docs/xml_results#glsp</span>
|
||||||
|
<span class="sd">.. _num: https://developers.google.com/custom-search/docs/xml_results#numsp</span>
|
||||||
|
<span class="sd">.. _save: https://developers.google.com/custom-search/docs/xml_results#safesp</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">locales</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="k">as</span> <span class="n">_fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">get_google_info</span><span class="p">,</span>
|
||||||
|
<span class="n">detect_google_sorry</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://news.google.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q12020'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developers.google.com/custom-search'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'news'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="c1"># Google-News results are always *SafeSearch*. Option 'safesearch' is set to</span>
|
||||||
|
<span class="c1"># False here, otherwise checker will report safesearch-errors::</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># safesearch : results are identical for safesearch=0 and safesearch=2</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="c1"># send_accept_language_header = True</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_news.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Google-News search request"""</span>
|
||||||
|
|
||||||
|
<span class="n">sxng_locale</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'searxng_locale'</span><span class="p">,</span> <span class="s1">'en-US'</span><span class="p">)</span>
|
||||||
|
<span class="n">ceid</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">get_engine_locale</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'ceid'</span><span class="p">],</span> <span class="n">default</span><span class="o">=</span><span class="s1">'US:en'</span><span class="p">)</span>
|
||||||
|
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'news.google.com'</span> <span class="c1"># google news has only one domain</span>
|
||||||
|
|
||||||
|
<span class="n">ceid_region</span><span class="p">,</span> <span class="n">ceid_lang</span> <span class="o">=</span> <span class="n">ceid</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">':'</span><span class="p">)</span>
|
||||||
|
<span class="n">ceid_lang</span><span class="p">,</span> <span class="n">ceid_suffix</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="n">ceid_lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="p">[</span>
|
||||||
|
<span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="p">)[:</span><span class="mi">2</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">ceid_suffix</span> <span class="ow">and</span> <span class="n">ceid_suffix</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'Hans'</span><span class="p">,</span> <span class="s1">'Hant'</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">ceid_region</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="n">ceid_lang</span><span class="p">:</span>
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">ceid_region</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">ceid_suffix</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">ceid_region</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="n">ceid_lang</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">ceid_region</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'AT'</span><span class="p">,</span> <span class="s1">'BE'</span><span class="p">,</span> <span class="s1">'CH'</span><span class="p">,</span> <span class="s1">'IL'</span><span class="p">,</span> <span class="s1">'SA'</span><span class="p">,</span> <span class="s1">'IN'</span><span class="p">,</span> <span class="s1">'BD'</span><span class="p">,</span> <span class="s1">'PT'</span><span class="p">]:</span>
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'hl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_lang</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">ceid_region</span>
|
||||||
|
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'lr'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'lang_'</span> <span class="o">+</span> <span class="n">ceid_lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">][</span><span class="s1">'gl'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ceid_region</span>
|
||||||
|
|
||||||
|
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s1">'https://'</span>
|
||||||
|
<span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"/search?"</span>
|
||||||
|
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="c1"># ceid includes a ':' character which must not be urlencoded</span>
|
||||||
|
<span class="o">+</span> <span class="p">(</span><span class="s1">'&ceid=</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">ceid</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_news.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get response from google's search request"""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># convert the text to dom</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[@class="xrnccd"]'</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="c1"># The first <a> tag in the <article> contains the link to the article</span>
|
||||||
|
<span class="c1"># The href attribute of the <a> tag is a google internal link, we have</span>
|
||||||
|
<span class="c1"># to decode</span>
|
||||||
|
|
||||||
|
<span class="n">href</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'./article/a/@href'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="n">href</span> <span class="o">=</span> <span class="n">href</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'?'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">href</span> <span class="o">=</span> <span class="n">href</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">href</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">urlsafe_b64decode</span><span class="p">(</span><span class="n">href</span> <span class="o">+</span> <span class="s1">'===='</span><span class="p">)</span>
|
||||||
|
<span class="n">href</span> <span class="o">=</span> <span class="n">href</span><span class="p">[</span><span class="n">href</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="sa">b</span><span class="s1">'http'</span><span class="p">)</span> <span class="p">:]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">b</span><span class="s1">'</span><span class="se">\xd2</span><span class="s1">'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">href</span> <span class="o">=</span> <span class="n">href</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'./article/h3[1]'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># The pub_date is mostly a string like 'yesterday', not a real</span>
|
||||||
|
<span class="c1"># timezone date or time. Therefore we can't use publishedDate.</span>
|
||||||
|
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'./article//time'</span><span class="p">))</span>
|
||||||
|
<span class="n">pub_origin</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'./article//a[@data-n-tid]'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="s1">' / '</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">pub_origin</span><span class="p">,</span> <span class="n">pub_date</span><span class="p">]</span> <span class="k">if</span> <span class="n">x</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="c1"># The image URL is located in a preceding sibling <img> tag, e.g.:</span>
|
||||||
|
<span class="c1"># "https://lh3.googleusercontent.com/DjhQh7DMszk.....z=-p-h100-w100"</span>
|
||||||
|
<span class="c1"># These URL are long but not personalized (double checked via tor).</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'preceding-sibling::a/figure/img/@src'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">href</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># return results</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">ceid_list</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'AE:ar'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'AR:es-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'AT:de'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'AU:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'BD:bn'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'BE:fr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'BE:nl'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'BG:bg'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'BR:pt-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'BW:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CA:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CA:fr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CH:de'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CH:fr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CL:es-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CN:zh-Hans'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CO:es-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CU:es-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'CZ:cs'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'DE:de'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'EG:ar'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ES:es'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ET:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'FR:fr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'GB:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'GH:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'GR:el'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'HK:zh-Hant'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'HU:hu'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ID:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ID:id'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IE:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IL:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IL:he'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IN:bn'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IN:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IN:hi'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IN:ml'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IN:mr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IN:ta'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IN:te'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'IT:it'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'JP:ja'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'KE:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'KR:ko'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'LB:ar'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'LT:lt'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'LV:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'LV:lv'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'MA:fr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'MX:es-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'MY:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'NA:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'NG:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'NL:nl'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'NO:no'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'NZ:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'PE:es-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'PH:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'PK:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'PL:pl'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'PT:pt-150'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'RO:ro'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'RS:sr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'RU:ru'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'SA:ar'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'SE:sv'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'SG:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'SI:sl'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'SK:sk'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'SN:fr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'TH:th'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'TR:tr'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'TW:zh-Hant'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'TZ:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'UA:ru'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'UA:uk'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'UG:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'US:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'US:es-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'VE:es-419'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'VN:vi'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ZA:en'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ZW:en'</span><span class="p">,</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="sd">"""List of region/language combinations supported by Google News. Values of the</span>
|
||||||
|
<span class="sd">``ceid`` argument of the Google News REST API."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">_skip_values</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'ET:en'</span><span class="p">,</span> <span class="c1"># english (ethiopia)</span>
|
||||||
|
<span class="s1">'ID:en'</span><span class="p">,</span> <span class="c1"># english (indonesia)</span>
|
||||||
|
<span class="s1">'LV:en'</span><span class="p">,</span> <span class="c1"># english (latvia)</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">_ceid_locale_map</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'NO:no'</span><span class="p">:</span> <span class="s1">'nb-NO'</span><span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="n">_fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">,</span> <span class="n">add_domains</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'ceid'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">ceid</span> <span class="ow">in</span> <span class="n">ceid_list</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">ceid</span> <span class="ow">in</span> <span class="n">_skip_values</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">region</span><span class="p">,</span> <span class="n">lang</span> <span class="o">=</span> <span class="n">ceid</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">':'</span><span class="p">)</span>
|
||||||
|
<span class="n">x</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'Hant'</span><span class="p">,</span> <span class="s1">'Hans'</span><span class="p">]:</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">sxng_locale</span> <span class="o">=</span> <span class="n">_ceid_locale_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">ceid</span><span class="p">,</span> <span class="n">lang</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">region</span><span class="p">)</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: </span><span class="si">%s</span><span class="s2"> -> </span><span class="si">%s</span><span class="s2"> is unknown by babel"</span> <span class="o">%</span> <span class="p">(</span><span class="n">ceid</span><span class="p">,</span> <span class="n">sxng_locale</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'ceid'</span><span class="p">][</span><span class="n">locales</span><span class="o">.</span><span class="n">region_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">ceid</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
364
_modules/searx/engines/google_scholar.html
Normal file
@@ -0,0 +1,364 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.google_scholar — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.google_scholar</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.google_scholar</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Google Scholar is a freely accessible web search engine that indexes the full</span>
|
||||||
|
<span class="sd">text or metadata of scholarly literature across an array of publishing formats</span>
|
||||||
|
<span class="sd">and disciplines.</span>
|
||||||
|
|
||||||
|
<span class="sd">Compared to other Google services the Scholar engine has a simple GET REST-API</span>
|
||||||
|
<span class="sd">and there does not exists ``async`` API. Even though the API slightly vintage</span>
|
||||||
|
<span class="sd">we can make use of the :ref:`google API` to assemble the arguments of the GET</span>
|
||||||
|
<span class="sd">request.</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: google scholar</span>
|
||||||
|
<span class="sd"> engine: google_scholar</span>
|
||||||
|
<span class="sd"> shortcut: gos</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">httpx</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">,</span>
|
||||||
|
<span class="n">ElementType</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span><span class="p">,</span> <span class="n">SearxEngineAccessDeniedException</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">get_google_info</span><span class="p">,</span>
|
||||||
|
<span class="n">time_range_dict</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://scholar.google.com"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q494817"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"https://developers.google.com/custom-search"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"HTML"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"science"</span><span class="p">,</span> <span class="s2">"scientific publications"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">50</span>
|
||||||
|
<span class="sd">"""`Google max 50 pages`_</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Google max 50 pages: https://github.com/searxng/searxng/issues/2982</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="n">language_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Google-Scholar search request"""</span>
|
||||||
|
|
||||||
|
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
|
||||||
|
<span class="c1"># subdomain is: scholar.google.xy</span>
|
||||||
|
<span class="n">google_info</span><span class="p">[</span><span class="s2">"subdomain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s2">"subdomain"</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"www."</span><span class="p">,</span> <span class="s2">"scholar."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"q"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s2">"params"</span><span class="p">],</span>
|
||||||
|
<span class="s2">"start"</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span><span class="p">,</span>
|
||||||
|
<span class="s2">"as_sdt"</span><span class="p">:</span> <span class="s2">"2007"</span><span class="p">,</span> <span class="c1"># include patents / to disable set "0,5"</span>
|
||||||
|
<span class="s2">"as_vis"</span><span class="p">:</span> <span class="s2">"0"</span><span class="p">,</span> <span class="c1"># include citations / to disable set "1"</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">args</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">time_range_args</span><span class="p">(</span><span class="n">params</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"https://"</span> <span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s2">"subdomain"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"/scholar?"</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">]</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">])</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span> <span class="c1"># pylint: disable=too-many-locals</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse response from Google Scholar"""</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">301</span><span class="p">,</span> <span class="mi">302</span><span class="p">,</span> <span class="mi">303</span><span class="p">,</span> <span class="mi">307</span><span class="p">,</span> <span class="mi">308</span><span class="p">)</span> <span class="ow">and</span> <span class="s2">"Location"</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="s2">"/sorry/index?continue"</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s2">"Location"</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># Our systems have detected unusual traffic from your computer</span>
|
||||||
|
<span class="c1"># network. Please try again later.</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineAccessDeniedException</span><span class="p">(</span>
|
||||||
|
<span class="n">message</span><span class="o">=</span><span class="s2">"google_scholar: unusual traffic detected"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">raise</span> <span class="n">httpx</span><span class="o">.</span><span class="n">TooManyRedirects</span><span class="p">(</span><span class="sa">f</span><span class="s2">"location </span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Location'</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'?'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="n">detect_google_captcha</span><span class="p">(</span><span class="n">dom</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse results</span>
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[@data-rp]"</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//h3[1]//a"</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">title</span><span class="p">:</span>
|
||||||
|
<span class="c1"># this is a [ZITATION] block</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">pub_type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//span[@class='gs_ctg2']"</span><span class="p">))</span> <span class="ow">or</span> <span class="s2">""</span>
|
||||||
|
<span class="k">if</span> <span class="n">pub_type</span><span class="p">:</span>
|
||||||
|
<span class="n">pub_type</span> <span class="o">=</span> <span class="n">pub_type</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//h3[1]//a/@href"</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//div[@class='gs_rs']"</span><span class="p">))</span> <span class="ow">or</span> <span class="s2">""</span>
|
||||||
|
<span class="n">authors</span><span class="p">,</span> <span class="n">journal</span><span class="p">,</span> <span class="n">publisher</span><span class="p">,</span> <span class="n">publishedDate</span> <span class="o">=</span> <span class="n">parse_gs_a</span><span class="p">(</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//div[@class='gs_a']"</span><span class="p">))</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">publisher</span> <span class="ow">in</span> <span class="n">url</span><span class="p">:</span>
|
||||||
|
<span class="n">publisher</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="c1"># cited by</span>
|
||||||
|
<span class="n">comments</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//div[@class='gs_fl']/a[starts-with(@href,'/scholar?cites=')]"</span><span class="p">))</span> <span class="ow">or</span> <span class="s2">""</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># link to the html or pdf document</span>
|
||||||
|
<span class="n">html_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">pdf_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">doc_url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//div[@class='gs_or_ggsm']/a/@href"</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">doc_type</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">".//span[@class='gs_ctg2']"</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">doc_type</span> <span class="o">==</span> <span class="s2">"[PDF]"</span><span class="p">:</span>
|
||||||
|
<span class="n">pdf_url</span> <span class="o">=</span> <span class="n">doc_url</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">html_url</span> <span class="o">=</span> <span class="n">doc_url</span>
|
||||||
|
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span>
|
||||||
|
<span class="nb">type</span><span class="o">=</span><span class="n">pub_type</span><span class="p">,</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="n">authors</span><span class="o">=</span><span class="n">authors</span><span class="p">,</span>
|
||||||
|
<span class="n">publisher</span><span class="o">=</span><span class="n">publisher</span><span class="p">,</span>
|
||||||
|
<span class="n">journal</span><span class="o">=</span><span class="n">journal</span><span class="p">,</span>
|
||||||
|
<span class="n">publishedDate</span><span class="o">=</span><span class="n">publishedDate</span><span class="p">,</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="n">comments</span><span class="o">=</span><span class="n">comments</span><span class="p">,</span>
|
||||||
|
<span class="n">html_url</span><span class="o">=</span><span class="n">html_url</span><span class="p">,</span>
|
||||||
|
<span class="n">pdf_url</span><span class="o">=</span><span class="n">pdf_url</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse suggestion</span>
|
||||||
|
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[contains(@class, 'gs_qsuggest_wrap')]//li//a"</span><span class="p">):</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="n">suggestion</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">correction</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[@class='gs_r gs_pda']/a"</span><span class="p">):</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="n">correction</span><span class="o">=</span><span class="n">extract_text</span><span class="p">(</span><span class="n">correction</span><span class="p">)))</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="time_range_args">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.time_range_args">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">time_range_args</span><span class="p">(</span><span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a dictionary with a time range arguments based on</span>
|
||||||
|
<span class="sd"> ``params["time_range"]``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Google Scholar supports a detailed search by year. Searching by *last</span>
|
||||||
|
<span class="sd"> month* or *last week* (as offered by SearXNG) is uncommon for scientific</span>
|
||||||
|
<span class="sd"> publications and is not supported by Google Scholar.</span>
|
||||||
|
|
||||||
|
<span class="sd"> To limit the result list when the users selects a range, all the SearXNG</span>
|
||||||
|
<span class="sd"> ranges (*day*, *week*, *month*, *year*) are mapped to *year*. If no range</span>
|
||||||
|
<span class="sd"> is set an empty dictionary of arguments is returned.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Example; when user selects a time range and we find ourselves in the year</span>
|
||||||
|
<span class="sd"> 2025 (current year minus one):</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> { "as_ylo" : 2024 }</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s2">"time_range"</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">[</span><span class="s2">"as_ylo"</span><span class="p">]</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">year</span> <span class="o">-</span> <span class="mi">1</span>
|
||||||
|
<span class="k">return</span> <span class="n">ret_val</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="detect_google_captcha">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.detect_google_captcha">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">detect_google_captcha</span><span class="p">(</span><span class="n">dom</span><span class="p">:</span> <span class="n">ElementType</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""In case of CAPTCHA Google Scholar open its own *not a Robot* dialog and is</span>
|
||||||
|
<span class="sd"> not redirected to ``sorry.google.com``.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//form[@id='gs_captcha_f']"</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">(</span><span class="n">message</span><span class="o">=</span><span class="s2">"CAPTCHA (gs_captcha_f)"</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="parse_gs_a">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_scholar.parse_gs_a">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_gs_a</span><span class="p">(</span><span class="n">text</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse the text written in green.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Possible formats:</span>
|
||||||
|
<span class="sd"> * "{authors} - {journal}, {year} - {publisher}"</span>
|
||||||
|
<span class="sd"> * "{authors} - {year} - {publisher}"</span>
|
||||||
|
<span class="sd"> * "{authors} - {publisher}"</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="n">text</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">text</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="p">[],</span> <span class="s2">""</span><span class="p">,</span> <span class="s2">""</span><span class="p">,</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">s_text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">" - "</span><span class="p">)</span>
|
||||||
|
<span class="n">authors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">s_text</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">", "</span><span class="p">)</span>
|
||||||
|
<span class="n">publisher</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">s_text</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">s_text</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">3</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">authors</span><span class="p">,</span> <span class="s2">""</span><span class="p">,</span> <span class="n">publisher</span><span class="p">,</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># the format is "{authors} - {journal}, {year} - {publisher}" or "{authors} - {year} - {publisher}"</span>
|
||||||
|
<span class="c1"># get journal and year</span>
|
||||||
|
<span class="n">journal_year</span> <span class="o">=</span> <span class="n">s_text</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">", "</span><span class="p">)</span>
|
||||||
|
<span class="c1"># journal is optional and may contains some coma</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">journal_year</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">journal</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">journal_year</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
|
||||||
|
<span class="k">if</span> <span class="n">journal</span> <span class="o">==</span> <span class="s2">"…"</span><span class="p">:</span>
|
||||||
|
<span class="n">journal</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">journal</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="c1"># year</span>
|
||||||
|
<span class="n">year</span> <span class="o">=</span> <span class="n">journal_year</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">publishedDate</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">year</span><span class="o">.</span><span class="n">strip</span><span class="p">(),</span> <span class="s2">"%Y"</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||||
|
<span class="n">publishedDate</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">return</span> <span class="n">authors</span><span class="p">,</span> <span class="n">journal</span><span class="p">,</span> <span class="n">publisher</span><span class="p">,</span> <span class="n">publishedDate</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
274
_modules/searx/engines/google_videos.html
Normal file
@@ -0,0 +1,274 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.google_videos — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.google_videos</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.google_videos</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This is the implementation of the Google Videos engine.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. admonition:: Content-Security-Policy (CSP)</span>
|
||||||
|
|
||||||
|
<span class="sd"> This engine needs to allow images from the `data URLs`_ (prefixed with the</span>
|
||||||
|
<span class="sd"> ``data:`` scheme)::</span>
|
||||||
|
|
||||||
|
<span class="sd"> Header set Content-Security-Policy "img-src 'self' data: ;"</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _data URLs:</span>
|
||||||
|
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">urlparse</span><span class="p">,</span> <span class="n">parse_qs</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.google</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">get_google_info</span><span class="p">,</span>
|
||||||
|
<span class="n">time_range_dict</span><span class="p">,</span>
|
||||||
|
<span class="n">filter_mapping</span><span class="p">,</span>
|
||||||
|
<span class="n">suggestion_xpath</span><span class="p">,</span>
|
||||||
|
<span class="n">detect_google_sorry</span><span class="p">,</span>
|
||||||
|
<span class="n">ui_async</span><span class="p">,</span>
|
||||||
|
<span class="n">parse_data_images</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_embeded_stream_url</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.google.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q219885'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developers.google.com/custom-search'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'videos'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">50</span>
|
||||||
|
<span class="n">language_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_videos.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Google-Video search request"""</span>
|
||||||
|
<span class="n">google_info</span> <span class="o">=</span> <span class="n">get_google_info</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">traits</span><span class="p">)</span>
|
||||||
|
<span class="n">start</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span>
|
||||||
|
|
||||||
|
<span class="n">query_url</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s1">'https://'</span>
|
||||||
|
<span class="o">+</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'subdomain'</span><span class="p">]</span>
|
||||||
|
<span class="o">+</span> <span class="s1">'/search'</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"?"</span>
|
||||||
|
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'tbm'</span><span class="p">:</span> <span class="s2">"vid"</span><span class="p">,</span>
|
||||||
|
<span class="s1">'start'</span><span class="p">:</span> <span class="n">start</span><span class="p">,</span>
|
||||||
|
<span class="o">**</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'params'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'asearch'</span><span class="p">:</span> <span class="s1">'arc'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'async'</span><span class="p">:</span> <span class="n">ui_async</span><span class="p">(</span><span class="n">start</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||||
|
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'tbs'</span><span class="p">:</span> <span class="s1">'qdr:'</span> <span class="o">+</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]})</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'safesearch'</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
|
||||||
|
<span class="n">query_url</span> <span class="o">+=</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'safe'</span><span class="p">:</span> <span class="n">filter_mapping</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]})</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query_url</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span> <span class="o">=</span> <span class="n">google_info</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">google_info</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">])</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/google.html#searx.engines.google_videos.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get response from google's search request"""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">detect_google_sorry</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
<span class="n">data_image_map</span> <span class="o">=</span> <span class="n">parse_data_images</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># convert the text to dom</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">result_divs</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class, "MjjYud")]'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse results</span>
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">result_divs</span><span class="p">:</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//h3[contains(@class, "LC20lb")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//a[@jsname="UWckNb"]/@href'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "ITZIwc")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">pub_info</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "gqF9jc")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="c1"># Broader XPath to find any <img> element</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//img/@src'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">duration</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//span[contains(@class, "k1U36b")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">video_id</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[@jscontroller="rTuANe"]/@data-vid'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Fallback for video_id from URL if not found via XPath</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">video_id</span> <span class="ow">and</span> <span class="n">url</span> <span class="ow">and</span> <span class="s1">'youtube.com'</span> <span class="ow">in</span> <span class="n">url</span><span class="p">:</span>
|
||||||
|
<span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
<span class="n">video_id</span> <span class="o">=</span> <span class="n">parse_qs</span><span class="p">(</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">query</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'v'</span><span class="p">,</span> <span class="p">[</span><span class="kc">None</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Handle thumbnail</span>
|
||||||
|
<span class="k">if</span> <span class="n">thumbnail</span> <span class="ow">and</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'data:image'</span><span class="p">):</span>
|
||||||
|
<span class="n">img_id</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//img/@id'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">img_id</span> <span class="ow">and</span> <span class="n">img_id</span> <span class="ow">in</span> <span class="n">data_image_map</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">data_image_map</span><span class="p">[</span><span class="n">img_id</span><span class="p">]</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">thumbnail</span> <span class="ow">and</span> <span class="n">video_id</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://img.youtube.com/vi/</span><span class="si">{</span><span class="n">video_id</span><span class="si">}</span><span class="s2">/hqdefault.jpg"</span>
|
||||||
|
|
||||||
|
<span class="c1"># Handle video embed URL</span>
|
||||||
|
<span class="n">embed_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">video_id</span><span class="p">:</span>
|
||||||
|
<span class="n">embed_url</span> <span class="o">=</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="sa">f</span><span class="s2">"https://www.youtube.com/watch?v=</span><span class="si">{</span><span class="n">video_id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">elif</span> <span class="n">url</span><span class="p">:</span>
|
||||||
|
<span class="n">embed_url</span> <span class="o">=</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Only append results with valid title and url</span>
|
||||||
|
<span class="k">if</span> <span class="n">title</span> <span class="ow">and</span> <span class="n">url</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
|
||||||
|
<span class="s1">'author'</span><span class="p">:</span> <span class="n">pub_info</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="s1">'length'</span><span class="p">:</span> <span class="n">duration</span><span class="p">,</span>
|
||||||
|
<span class="s1">'iframe_src'</span><span class="p">:</span> <span class="n">embed_url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'videos.html'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse suggestion</span>
|
||||||
|
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
537
_modules/searx/engines/json_engine.html
Normal file
@@ -0,0 +1,537 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.json_engine — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.json_engine</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.json_engine</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""The JSON engine is a *generic* engine with which it is possible to configure</span>
|
||||||
|
<span class="sd">engines in the settings.</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">Request:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`search_url`</span>
|
||||||
|
<span class="sd">- :py:obj:`lang_all`</span>
|
||||||
|
<span class="sd">- :py:obj:`soft_max_redirects`</span>
|
||||||
|
<span class="sd">- :py:obj:`method`</span>
|
||||||
|
<span class="sd">- :py:obj:`request_body`</span>
|
||||||
|
<span class="sd">- :py:obj:`cookies`</span>
|
||||||
|
<span class="sd">- :py:obj:`headers`</span>
|
||||||
|
|
||||||
|
<span class="sd">Paging:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`paging`</span>
|
||||||
|
<span class="sd">- :py:obj:`page_size`</span>
|
||||||
|
<span class="sd">- :py:obj:`first_page_num`</span>
|
||||||
|
|
||||||
|
<span class="sd">Time Range:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`time_range_support`</span>
|
||||||
|
<span class="sd">- :py:obj:`time_range_url`</span>
|
||||||
|
<span class="sd">- :py:obj:`time_range_map`</span>
|
||||||
|
|
||||||
|
<span class="sd">Safe-Search:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`safe_search_support`</span>
|
||||||
|
<span class="sd">- :py:obj:`safe_search_map`</span>
|
||||||
|
|
||||||
|
<span class="sd">Response:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`title_html_to_text`</span>
|
||||||
|
<span class="sd">- :py:obj:`content_html_to_text`</span>
|
||||||
|
<span class="sd">- :py:obj:`no_result_for_http_status`</span>
|
||||||
|
|
||||||
|
<span class="sd">JSON query:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`results_query`</span>
|
||||||
|
<span class="sd">- :py:obj:`url_query`</span>
|
||||||
|
<span class="sd">- :py:obj:`url_prefix`</span>
|
||||||
|
<span class="sd">- :py:obj:`title_query`</span>
|
||||||
|
<span class="sd">- :py:obj:`content_query`</span>
|
||||||
|
<span class="sd">- :py:obj:`thumbnail_query`</span>
|
||||||
|
<span class="sd">- :py:obj:`thumbnail_prefix`</span>
|
||||||
|
<span class="sd">- :py:obj:`suggestion_query`</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Example</span>
|
||||||
|
<span class="sd">=======</span>
|
||||||
|
|
||||||
|
<span class="sd">Here is a simple example of a JSON engine configure in the :ref:`settings</span>
|
||||||
|
<span class="sd">engines` section, further read :ref:`engines-dev`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name : mdn</span>
|
||||||
|
<span class="sd"> engine : json_engine</span>
|
||||||
|
<span class="sd"> paging : True</span>
|
||||||
|
<span class="sd"> search_url : https://developer.mozilla.org/api/v1/search?q={query}&page={pageno}</span>
|
||||||
|
<span class="sd"> results_query : documents</span>
|
||||||
|
<span class="sd"> url_query : mdn_url</span>
|
||||||
|
<span class="sd"> url_prefix : https://developer.mozilla.org</span>
|
||||||
|
<span class="sd"> title_query : title</span>
|
||||||
|
<span class="sd"> content_query : summary</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Iterable</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">to_string</span><span class="p">,</span> <span class="n">html_to_text</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
|
||||||
|
|
||||||
|
<span class="n">search_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Search URL of the engine. Example::</span>
|
||||||
|
|
||||||
|
<span class="sd"> https://example.org/?search={query}&page={pageno}{time_range}{safe_search}</span>
|
||||||
|
|
||||||
|
<span class="sd">Replacements are:</span>
|
||||||
|
|
||||||
|
<span class="sd">``{query}``:</span>
|
||||||
|
<span class="sd"> Search terms from user.</span>
|
||||||
|
|
||||||
|
<span class="sd">``{pageno}``:</span>
|
||||||
|
<span class="sd"> Page number if engine supports paging :py:obj:`paging`</span>
|
||||||
|
|
||||||
|
<span class="sd">``{lang}``:</span>
|
||||||
|
<span class="sd"> ISO 639-1 language code (en, de, fr ..)</span>
|
||||||
|
|
||||||
|
<span class="sd">``{time_range}``:</span>
|
||||||
|
<span class="sd"> :py:obj:`URL parameter <time_range_url>` if engine :py:obj:`supports time</span>
|
||||||
|
<span class="sd"> range <time_range_support>`. The value for the parameter is taken from</span>
|
||||||
|
<span class="sd"> :py:obj:`time_range_map`.</span>
|
||||||
|
|
||||||
|
<span class="sd">``{safe_search}``:</span>
|
||||||
|
<span class="sd"> Safe-search :py:obj:`URL parameter <safe_search_map>` if engine</span>
|
||||||
|
<span class="sd"> :py:obj:`supports safe-search <safe_search_support>`. The ``{safe_search}``</span>
|
||||||
|
<span class="sd"> replacement is taken from the :py:obj:`safes_search_map`. Filter results::</span>
|
||||||
|
|
||||||
|
<span class="sd"> 0: none, 1: moderate, 2:strict</span>
|
||||||
|
|
||||||
|
<span class="sd"> If not supported, the URL parameter is an empty string.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">lang_all</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||||
|
<span class="sd">'''Replacement ``{lang}`` in :py:obj:`search_url` if language ``all`` is</span>
|
||||||
|
<span class="sd">selected.</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">no_result_for_http_status</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="sd">'''Return empty result for these HTTP status codes instead of throwing an error.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> no_result_for_http_status: []</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">soft_max_redirects</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="sd">'''Maximum redirects, soft limit. Record an error but don't stop the engine'''</span>
|
||||||
|
|
||||||
|
<span class="n">method</span> <span class="o">=</span> <span class="s1">'GET'</span>
|
||||||
|
<span class="sd">'''Some engines might require to do POST requests for search.'''</span>
|
||||||
|
|
||||||
|
<span class="n">request_body</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="sd">'''The body of the request. This can only be used if different :py:obj:`method`</span>
|
||||||
|
<span class="sd">is set, e.g. ``POST``. For formatting see the documentation of :py:obj:`search_url`.</span>
|
||||||
|
|
||||||
|
<span class="sd">Note: Curly brackets which aren't encapsulating a replacement placeholder</span>
|
||||||
|
<span class="sd">must be escaped by doubling each ``{`` and ``}``.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> request_body: >-</span>
|
||||||
|
<span class="sd"> {{</span>
|
||||||
|
<span class="sd"> "search": "{query}",</span>
|
||||||
|
<span class="sd"> "page": {pageno},</span>
|
||||||
|
<span class="sd"> "extra": {{</span>
|
||||||
|
<span class="sd"> "time_range": {time_range},</span>
|
||||||
|
<span class="sd"> "rating": "{safe_search}"</span>
|
||||||
|
<span class="sd"> }}</span>
|
||||||
|
<span class="sd"> }}</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">cookies</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="sd">'''Some engines might offer different result based on cookies.</span>
|
||||||
|
<span class="sd">Possible use-case: To set safesearch cookie.'''</span>
|
||||||
|
|
||||||
|
<span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="sd">'''Some engines might offer different result based on cookies or headers.</span>
|
||||||
|
<span class="sd">Possible use-case: To set safesearch cookie or header to moderate.'''</span>
|
||||||
|
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''Engine supports paging [True or False].'''</span>
|
||||||
|
|
||||||
|
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">1</span>
|
||||||
|
<span class="sd">'''Number of results on each page. Only needed if the site requires not a page</span>
|
||||||
|
<span class="sd">number, but an offset.'''</span>
|
||||||
|
|
||||||
|
<span class="n">first_page_num</span> <span class="o">=</span> <span class="mi">1</span>
|
||||||
|
<span class="sd">'''Number of the first page (usually 0 or 1).'''</span>
|
||||||
|
|
||||||
|
<span class="n">results_query</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="sd">'''JSON query for the list of result items.</span>
|
||||||
|
|
||||||
|
<span class="sd">The query string is a slash `/` separated path of JSON key names.</span>
|
||||||
|
<span class="sd">Array entries can be specified using the index or can be omitted entirely,</span>
|
||||||
|
<span class="sd">in which case each entry is considered -</span>
|
||||||
|
<span class="sd">most implementations will default to the first entry in this case.</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">url_query</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">'''JSON query of result's ``url``. For the query string documentation see :py:obj:`results_query`'''</span>
|
||||||
|
|
||||||
|
<span class="n">url_prefix</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">'''String to prepend to the result's ``url``.'''</span>
|
||||||
|
|
||||||
|
<span class="n">title_query</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">'''JSON query of result's ``title``. For the query string documentation see :py:obj:`results_query`'''</span>
|
||||||
|
|
||||||
|
<span class="n">content_query</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">'''JSON query of result's ``content``. For the query string documentation see :py:obj:`results_query`'''</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail_query</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''JSON query of result's ``thumbnail``. For the query string documentation see :py:obj:`results_query`'''</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail_prefix</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="sd">'''String to prepend to the result's ``thumbnail``.'''</span>
|
||||||
|
|
||||||
|
<span class="n">suggestion_query</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="sd">'''JSON query of result's ``suggestion``. For the query string documentation see :py:obj:`results_query`'''</span>
|
||||||
|
|
||||||
|
<span class="n">title_html_to_text</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''Extract text from a HTML title string'''</span>
|
||||||
|
|
||||||
|
<span class="n">content_html_to_text</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''Extract text from a HTML content string'''</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''Engine supports search time range.'''</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_url</span> <span class="o">=</span> <span class="s1">'&hours=</span><span class="si">{time_range_val}</span><span class="s1">'</span>
|
||||||
|
<span class="sd">'''Time range URL parameter in the in :py:obj:`search_url`. If no time range is</span>
|
||||||
|
<span class="sd">requested by the user, the URL parameter is an empty string. The</span>
|
||||||
|
<span class="sd">``{time_range_val}`` replacement is taken from the :py:obj:`time_range_map`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> time_range_url : '&days={time_range_val}'</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'day'</span><span class="p">:</span> <span class="mi">24</span><span class="p">,</span>
|
||||||
|
<span class="s1">'week'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
|
||||||
|
<span class="s1">'month'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">30</span><span class="p">,</span>
|
||||||
|
<span class="s1">'year'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">365</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">'''Maps time range value from user to ``{time_range_val}`` in</span>
|
||||||
|
<span class="sd">:py:obj:`time_range_url`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> time_range_map:</span>
|
||||||
|
<span class="sd"> day: 1</span>
|
||||||
|
<span class="sd"> week: 7</span>
|
||||||
|
<span class="sd"> month: 30</span>
|
||||||
|
<span class="sd"> year: 365</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">safe_search_support</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''Engine supports safe-search.'''</span>
|
||||||
|
|
||||||
|
<span class="n">safe_search_map</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'&filter=none'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'&filter=moderate'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'&filter=strict'</span><span class="p">}</span>
|
||||||
|
<span class="sd">'''Maps safe-search value to ``{safe_search}`` in :py:obj:`search_url`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> safesearch: true</span>
|
||||||
|
<span class="sd"> safes_search_map:</span>
|
||||||
|
<span class="sd"> 0: '&filter=none'</span>
|
||||||
|
<span class="sd"> 1: '&filter=moderate'</span>
|
||||||
|
<span class="sd"> 2: '&filter=strict'</span>
|
||||||
|
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">iterate</span><span class="p">(</span><span class="n">iterable</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">iterable</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="n">items</span> <span class="o">=</span> <span class="n">iterable</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">items</span> <span class="o">=</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">iterable</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">index</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">items</span><span class="p">:</span>
|
||||||
|
<span class="k">yield</span> <span class="nb">str</span><span class="p">(</span><span class="n">index</span><span class="p">),</span> <span class="n">value</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_iterable</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">Iterable</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse</span><span class="p">(</span><span class="n">query</span><span class="p">):</span> <span class="c1"># pylint: disable=redefined-outer-name</span>
|
||||||
|
<span class="n">q</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'/'</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">part</span> <span class="o">==</span> <span class="s1">''</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">q</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">part</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">q</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">do_query</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">q</span><span class="p">):</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
<span class="n">ret</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">q</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">ret</span>
|
||||||
|
|
||||||
|
<span class="n">qkey</span> <span class="o">=</span> <span class="n">q</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">iterate</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">q</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">key</span> <span class="o">==</span> <span class="n">qkey</span><span class="p">:</span>
|
||||||
|
<span class="n">ret</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">elif</span> <span class="n">is_iterable</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
|
||||||
|
<span class="n">ret</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">do_query</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">q</span><span class="p">))</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_iterable</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="n">key</span> <span class="o">==</span> <span class="n">qkey</span><span class="p">:</span>
|
||||||
|
<span class="n">ret</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">do_query</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">q</span><span class="p">[</span><span class="mi">1</span><span class="p">:]))</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">ret</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">do_query</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">q</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">ret</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">query</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">query_string</span><span class="p">):</span>
|
||||||
|
<span class="n">q</span> <span class="o">=</span> <span class="n">parse</span><span class="p">(</span><span class="n">query_string</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">do_query</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">q</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/json_engine.html#searx.engines.json_engine.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span> <span class="c1"># pylint: disable=redefined-outer-name</span>
|
||||||
|
<span class="w"> </span><span class="sd">'''Build request parameters (see :ref:`engine request`).'''</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang_all</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'all'</span><span class="p">:</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">][:</span><span class="mi">2</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">time_range</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'time_range'</span><span class="p">):</span>
|
||||||
|
<span class="n">time_range_val</span> <span class="o">=</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'time_range'</span><span class="p">))</span>
|
||||||
|
<span class="n">time_range</span> <span class="o">=</span> <span class="n">time_range_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">time_range_val</span><span class="o">=</span><span class="n">time_range_val</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">safe_search</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]:</span>
|
||||||
|
<span class="n">safe_search</span> <span class="o">=</span> <span class="n">safe_search_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="n">fp</span> <span class="o">=</span> <span class="p">{</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
<span class="s1">'query'</span><span class="p">:</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">})[</span><span class="mi">2</span><span class="p">:],</span>
|
||||||
|
<span class="s1">'lang'</span><span class="p">:</span> <span class="n">lang</span><span class="p">,</span>
|
||||||
|
<span class="s1">'pageno'</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">page_size</span> <span class="o">+</span> <span class="n">first_page_num</span><span class="p">,</span>
|
||||||
|
<span class="s1">'time_range'</span><span class="p">:</span> <span class="n">time_range</span><span class="p">,</span>
|
||||||
|
<span class="s1">'safe_search'</span><span class="p">:</span> <span class="n">safe_search</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">cookies</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fp</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'method'</span><span class="p">]</span> <span class="o">=</span> <span class="n">method</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">request_body</span><span class="p">:</span>
|
||||||
|
<span class="c1"># don't url-encode the query if it's in the request body</span>
|
||||||
|
<span class="n">fp</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">]</span> <span class="o">=</span> <span class="n">request_body</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'soft_max_redirects'</span><span class="p">]</span> <span class="o">=</span> <span class="n">soft_max_redirects</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">identity</span><span class="p">(</span><span class="n">arg</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">arg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">extract_response_info</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
|
||||||
|
<span class="n">title_filter</span> <span class="o">=</span> <span class="n">html_to_text</span> <span class="k">if</span> <span class="n">title_html_to_text</span> <span class="k">else</span> <span class="n">identity</span>
|
||||||
|
<span class="n">content_filter</span> <span class="o">=</span> <span class="n">html_to_text</span> <span class="k">if</span> <span class="n">content_html_to_text</span> <span class="k">else</span> <span class="n">identity</span>
|
||||||
|
|
||||||
|
<span class="n">tmp_result</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">url_query</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">url_prefix</span> <span class="o">+</span> <span class="n">to_string</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_query</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]</span> <span class="o">=</span> <span class="n">title_filter</span><span class="p">(</span><span class="n">to_string</span><span class="p">(</span><span class="n">title</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span><span class="p">:</span> <span class="c1"># pylint: disable=bare-except</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">content_query</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'content'</span><span class="p">]</span> <span class="o">=</span> <span class="n">content_filter</span><span class="p">(</span><span class="n">to_string</span><span class="p">(</span><span class="n">content</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span><span class="p">:</span> <span class="c1"># pylint: disable=bare-except</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'content'</span><span class="p">]</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">thumbnail_query</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail_query_result</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">thumbnail_query</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'thumbnail'</span><span class="p">]</span> <span class="o">=</span> <span class="n">thumbnail_prefix</span> <span class="o">+</span> <span class="n">to_string</span><span class="p">(</span><span class="n">thumbnail_query_result</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span><span class="p">:</span> <span class="c1"># pylint: disable=bare-except</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">tmp_result</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/json_engine.html#searx.engines.json_engine.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">'''Scrap *results* from the response (see :ref:`result types`).'''</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">no_result_for_http_status</span> <span class="ow">and</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="n">no_result_for_http_status</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">json</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="n">is_onion</span> <span class="o">=</span> <span class="s1">'onions'</span> <span class="ow">in</span> <span class="n">categories</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">results_query</span><span class="p">:</span>
|
||||||
|
<span class="n">rs</span> <span class="o">=</span> <span class="n">query</span><span class="p">(</span><span class="n">json</span><span class="p">,</span> <span class="n">results_query</span><span class="p">)</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">rs</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
<span class="n">rs</span> <span class="o">=</span> <span class="n">rs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">rs</span> <span class="o">=</span> <span class="n">json</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">rs</span><span class="p">:</span>
|
||||||
|
<span class="n">tmp_result</span> <span class="o">=</span> <span class="n">extract_response_info</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">tmp_result</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">is_onion</span><span class="p">:</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'is_onion'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tmp_result</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">suggestion_query</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">query</span><span class="p">(</span><span class="n">json</span><span class="p">,</span> <span class="n">suggestion_query</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">suggestion</span><span class="p">})</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
236
_modules/searx/engines/marginalia.html
Normal file
@@ -0,0 +1,236 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.marginalia — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.marginalia</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.marginalia</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""`Marginalia Search`_ is an independent open source Internet search engine</span>
|
||||||
|
<span class="sd">operating out of Sweden. It is principally developed and operated by Viktor</span>
|
||||||
|
<span class="sd">Lofgren .</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Marginalia Search:</span>
|
||||||
|
<span class="sd"> https://about.marginalia-search.com/</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following required settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`api_key`</span>
|
||||||
|
|
||||||
|
<span class="sd">You can configure a Marginalia engine by:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: marginalia</span>
|
||||||
|
<span class="sd"> engine: marginalia</span>
|
||||||
|
<span class="sd"> shortcut: mar</span>
|
||||||
|
<span class="sd"> api_key: ...</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">searxng_useragent</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://marginalia.nu"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"https://about.marginalia-search.com/article/api/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://api2.marginalia-search.com"</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"general"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">results_per_page</span> <span class="o">=</span> <span class="mi">20</span>
|
||||||
|
<span class="n">api_key</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">"""To get an API key, please follow the instructions from `Key and license`_</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Key and license:</span>
|
||||||
|
<span class="sd"> https://about.marginalia-search.com/article/api/</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ApiSearchResult">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/marginalia.html#searx.engines.marginalia.ApiSearchResult">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ApiSearchResult</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Marginalia's ApiSearchResult_ class definition.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _ApiSearchResult:</span>
|
||||||
|
<span class="sd"> https://github.com/MarginaliaSearch/MarginaliaSearch/blob/master/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResult.java</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">description</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">quality</span><span class="p">:</span> <span class="nb">float</span>
|
||||||
|
<span class="nb">format</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">details</span><span class="p">:</span> <span class="nb">str</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ApiSearchResults">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/marginalia.html#searx.engines.marginalia.ApiSearchResults">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ApiSearchResults</span><span class="p">(</span><span class="n">t</span><span class="o">.</span><span class="n">TypedDict</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Marginalia's ApiSearchResults_ class definition.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _ApiSearchResults:</span>
|
||||||
|
<span class="sd"> https://github.com/MarginaliaSearch/MarginaliaSearch/blob/master/code/services-application/api-service/java/nu/marginalia/api/model/ApiSearchResults.java</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">license</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">query</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">results</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="n">ApiSearchResult</span><span class="p">]</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
|
||||||
|
|
||||||
|
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"count"</span><span class="p">:</span> <span class="n">results_per_page</span><span class="p">,</span> <span class="s2">"nsfw"</span><span class="p">:</span> <span class="nb">min</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"safesearch"</span><span class="p">],</span> <span class="mi">1</span><span class="p">),</span> <span class="s2">"query"</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">][</span><span class="s2">"User-Agent"</span><span class="p">]</span> <span class="o">=</span> <span class="n">searxng_useragent</span><span class="p">()</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">][</span><span class="s2">"API-Key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="n">SXNG_Response</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">resp_json</span><span class="p">:</span> <span class="n">ApiSearchResults</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">resp_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"results"</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">item</span><span class="p">[</span><span class="s2">"title"</span><span class="p">],</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">item</span><span class="p">[</span><span class="s2">"url"</span><span class="p">],</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"description"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
|
||||||
|
|
||||||
|
<span class="n">_api_key</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"api_key"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">_api_key</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"missing api_key: see https://about.marginalia-search.com/article/api"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">_api_key</span> <span class="o">==</span> <span class="s2">"public"</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"invalid api_key (</span><span class="si">%s</span><span class="s2">): see https://about.marginalia-search.com/article/api"</span><span class="p">,</span> <span class="n">api_key</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
181
_modules/searx/engines/mrs.html
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.mrs — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.mrs</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.mrs</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Matrix Rooms Search - a fully-featured, standalone, matrix rooms search service.</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following mandatory settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`base_url`</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: MRS</span>
|
||||||
|
<span class="sd"> engine: mrs</span>
|
||||||
|
<span class="sd"> base_url: https://mrs-host</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementation</span>
|
||||||
|
<span class="sd">==============</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote_plus</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://matrixrooms.info'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://gitlab.com/etke.cc/mrs/api/-/blob/main/openapi.yml?ref_type=heads'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'social media'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">matrix_url</span> <span class="o">=</span> <span class="s2">"https://matrix.to"</span>
|
||||||
|
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">20</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="init">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/mrs.html#searx.engines.mrs.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The ``base_url`` must be set in the configuration, if ``base_url`` is not</span>
|
||||||
|
<span class="sd"> set, a :py:obj:`ValueError` is raised during initialization.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">base_url</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'engine MRS, base_url is unset'</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/search/</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">query</span><span class="p">)</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="n">page_size</span><span class="si">}</span><span class="s2">/</span><span class="si">{</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="n">page_size</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">():</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">matrix_url</span> <span class="o">+</span> <span class="s1">'/#/'</span> <span class="o">+</span> <span class="n">result</span><span class="p">[</span><span class="s1">'alias'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'topic'</span><span class="p">]</span>
|
||||||
|
<span class="o">+</span> <span class="sa">f</span><span class="s2">" // </span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">'members'</span><span class="p">]</span><span class="si">}</span><span class="s2"> members"</span>
|
||||||
|
<span class="o">+</span> <span class="sa">f</span><span class="s2">" // </span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">'alias'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="sa">f</span><span class="s2">" // </span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">'server'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'avatar_url'</span><span class="p">],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
253
_modules/searx/engines/odysee.html
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.odysee — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.odysee</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.odysee</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Odysee_ is a decentralized video hosting platform.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Odysee: https://github.com/OdyseeTeam/odysee-frontend</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
|
||||||
|
<span class="c1"># Engine metadata</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://odysee.com/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q102046570"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># Engine configuration</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">results_per_page</span> <span class="o">=</span> <span class="mi">20</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'videos'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Search URL (Note: lighthouse.lbry.com/search works too, and may be faster at times)</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://lighthouse.odysee.tv/search"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"day"</span><span class="p">:</span> <span class="s2">"today"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"week"</span><span class="p">:</span> <span class="s2">"thisweek"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"month"</span><span class="p">:</span> <span class="s2">"thismonth"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"year"</span><span class="p">:</span> <span class="s2">"thisyear"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">start_index</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">results_per_page</span>
|
||||||
|
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"s"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"size"</span><span class="p">:</span> <span class="n">results_per_page</span><span class="p">,</span>
|
||||||
|
<span class="s2">"from"</span><span class="p">:</span> <span class="n">start_index</span><span class="p">,</span>
|
||||||
|
<span class="s2">"include"</span><span class="p">:</span> <span class="s2">"channel,thumbnail_url,title,description,duration,release_time"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"mediaType"</span><span class="p">:</span> <span class="s2">"video"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">lang</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_dict</span><span class="p">:</span>
|
||||||
|
<span class="n">query_params</span><span class="p">[</span><span class="s1">'time_filter'</span><span class="p">]</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">query_params</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># Format the video duration</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">format_duration</span><span class="p">(</span><span class="n">duration</span><span class="p">):</span>
|
||||||
|
<span class="n">seconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">duration</span><span class="p">)</span>
|
||||||
|
<span class="n">length</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">gmtime</span><span class="p">(</span><span class="n">seconds</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">length</span><span class="o">.</span><span class="n">tm_hour</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%H:%M:%S"</span><span class="p">,</span> <span class="n">length</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">time</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%M:%S"</span><span class="p">,</span> <span class="n">length</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
|
||||||
|
<span class="n">claim_id</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"claimId"</span><span class="p">]</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"title"</span><span class="p">]</span>
|
||||||
|
<span class="n">thumbnail_url</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"thumbnail_url"</span><span class="p">]</span>
|
||||||
|
<span class="n">description</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"description"</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">""</span>
|
||||||
|
<span class="n">channel</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"channel"</span><span class="p">]</span>
|
||||||
|
<span class="n">release_time</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"release_time"</span><span class="p">]</span>
|
||||||
|
<span class="n">duration</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"duration"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">release_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">release_time</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"T"</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span> <span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">formatted_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">release_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">())</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://odysee.com/</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">claim_id</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">iframe_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://odysee.com/$/embed/</span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">claim_id</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">odysee_thumbnail</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://thumbnails.odycdn.com/optimize/s:390:0/quality:85/plain/</span><span class="si">{</span><span class="n">thumbnail_url</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">formatted_duration</span> <span class="o">=</span> <span class="n">format_duration</span><span class="p">(</span><span class="n">duration</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s2">"title"</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s2">"url"</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="s2">"content"</span><span class="p">:</span> <span class="n">description</span><span class="p">,</span>
|
||||||
|
<span class="s2">"author"</span><span class="p">:</span> <span class="n">channel</span><span class="p">,</span>
|
||||||
|
<span class="s2">"publishedDate"</span><span class="p">:</span> <span class="n">formatted_date</span><span class="p">,</span>
|
||||||
|
<span class="s2">"length"</span><span class="p">:</span> <span class="n">formatted_duration</span><span class="p">,</span>
|
||||||
|
<span class="s2">"thumbnail"</span><span class="p">:</span> <span class="n">odysee_thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="s2">"iframe_src"</span><span class="p">:</span> <span class="n">iframe_url</span><span class="p">,</span>
|
||||||
|
<span class="s2">"template"</span><span class="p">:</span> <span class="s2">"videos.html"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/odysee.html#searx.engines.odysee.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""</span>
|
||||||
|
<span class="sd"> Fetch languages from Odysee's source code.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span>
|
||||||
|
<span class="s1">'https://raw.githubusercontent.com/OdyseeTeam/odysee-frontend/master/ui/constants/supported_browser_languages.js'</span><span class="p">,</span> <span class="c1"># pylint: disable=line-too-long</span>
|
||||||
|
<span class="n">timeout</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: can't determine languages from Odysee"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">4</span><span class="p">]:</span>
|
||||||
|
<span class="n">lang_tag</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">": "</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">"-"</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: </span><span class="si">%s</span><span class="s2"> is unknown by babel"</span> <span class="o">%</span> <span class="n">lang_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">lang_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">lang_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang_tag</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
299
_modules/searx/engines/peertube.html
Normal file
@@ -0,0 +1,299 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.peertube — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.peertube</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.peertube</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Peertube and :py:obj:`SepiaSearch <searx.engines.sepiasearch>` do share</span>
|
||||||
|
<span class="sd">(more or less) the same REST API and the schema of the JSON result is identical.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.relativedelta</span><span class="w"> </span><span class="kn">import</span> <span class="n">relativedelta</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span><span class="p">,</span> <span class="n">humanize_number</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="c1"># pylint: disable=line-too-long</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://joinpeertube.org'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q50938515'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://docs.joinpeertube.org/api-rest-reference.html#tag/Search/operation/searchVideos'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"videos"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://peer.tube"</span>
|
||||||
|
<span class="sd">"""Base URL of the Peertube instance. A list of instances is available at:</span>
|
||||||
|
|
||||||
|
<span class="sd">- https://instances.joinpeertube.org/instances</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_table</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'day'</span><span class="p">:</span> <span class="n">relativedelta</span><span class="p">(),</span>
|
||||||
|
<span class="s1">'week'</span><span class="p">:</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">weeks</span><span class="o">=-</span><span class="mi">1</span><span class="p">),</span>
|
||||||
|
<span class="s1">'month'</span><span class="p">:</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">months</span><span class="o">=-</span><span class="mi">1</span><span class="p">),</span>
|
||||||
|
<span class="s1">'year'</span><span class="p">:</span> <span class="n">relativedelta</span><span class="p">(</span><span class="n">years</span><span class="o">=-</span><span class="mi">1</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch_table</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'both'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'false'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'false'</span><span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/peertube.html#searx.engines.peertube.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Assemble request for the Peertube API"""</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="c1"># eng_region = traits.get_region(params['searxng_locale'], 'en_US')</span>
|
||||||
|
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"/api/v1/search/videos?"</span>
|
||||||
|
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'search'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'searchTarget'</span><span class="p">:</span> <span class="s1">'search-index'</span><span class="p">,</span> <span class="c1"># Vidiversum</span>
|
||||||
|
<span class="s1">'resultType'</span><span class="p">:</span> <span class="s1">'videos'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'start'</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span><span class="p">,</span>
|
||||||
|
<span class="s1">'count'</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
|
||||||
|
<span class="c1"># -createdAt: sort by date ascending / createdAt: date descending</span>
|
||||||
|
<span class="s1">'sort'</span><span class="p">:</span> <span class="s1">'-match'</span><span class="p">,</span> <span class="c1"># sort by *match descending*</span>
|
||||||
|
<span class="s1">'nsfw'</span><span class="p">:</span> <span class="n">safesearch_table</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">eng_lang</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">'&languageOneOf[]='</span> <span class="o">+</span> <span class="n">eng_lang</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">'&boostLanguages[]='</span> <span class="o">+</span> <span class="n">eng_lang</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_table</span><span class="p">:</span>
|
||||||
|
<span class="n">time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">date</span><span class="p">()</span> <span class="o">+</span> <span class="n">time_range_table</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">'&startDate='</span> <span class="o">+</span> <span class="n">time</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="video_response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/peertube.html#searx.engines.peertube.video_response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse video response from SepiaSearch and Peertube instances."""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'data'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">'data'</span><span class="p">]:</span>
|
||||||
|
<span class="n">metadata</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="n">x</span>
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'channel'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'displayName'</span><span class="p">),</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'channel'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'name'</span><span class="p">)</span> <span class="o">+</span> <span class="s1">'@'</span> <span class="o">+</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'channel'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'host'</span><span class="p">),</span>
|
||||||
|
<span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tags'</span><span class="p">,</span> <span class="p">[])),</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">x</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">duration</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'duration'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">duration</span><span class="p">:</span>
|
||||||
|
<span class="n">duration</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="n">duration</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'description'</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">),</span>
|
||||||
|
<span class="s1">'author'</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'account'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'displayName'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'length'</span><span class="p">:</span> <span class="n">duration</span><span class="p">,</span>
|
||||||
|
<span class="s1">'views'</span><span class="p">:</span> <span class="n">humanize_number</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">'views'</span><span class="p">]),</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'videos.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">parse</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">'publishedAt'</span><span class="p">]),</span>
|
||||||
|
<span class="s1">'iframe_src'</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'embedUrl'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'thumbnailUrl'</span><span class="p">)</span> <span class="ow">or</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'previewUrl'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'metadata'</span><span class="p">:</span> <span class="s1">' | '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/peertube.html#searx.engines.peertube.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages from peertube's search-index source code.</span>
|
||||||
|
|
||||||
|
<span class="sd"> See videoLanguages_ in commit `8ed5c729 - Refactor and redesign client`_</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _8ed5c729 - Refactor and redesign client:</span>
|
||||||
|
<span class="sd"> https://framagit.org/framasoft/peertube/search-index/-/commit/8ed5c729</span>
|
||||||
|
<span class="sd"> .. _videoLanguages:</span>
|
||||||
|
<span class="sd"> https://framagit.org/framasoft/peertube/search-index/-/commit/8ed5c729#3d8747f9a60695c367c70bb64efba8f403721fad_0_291</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span>
|
||||||
|
<span class="s1">'https://framagit.org/framasoft/peertube/search-index/-/raw/master/client/src/components/Filters.vue'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># the response from search-index repository is very slow</span>
|
||||||
|
<span class="n">timeout</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from peertube is not OK."</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="n">js_lang</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="sa">r</span><span class="s2">"videoLanguages \(\)[^\n]+(.*?)\]"</span><span class="p">,</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">DOTALL</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">js_lang</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: can't determine languages from peertube"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">re</span><span class="o">.</span><span class="n">finditer</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\{ id: '([a-z]+)', label:"</span><span class="p">,</span> <span class="n">js_lang</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)):</span>
|
||||||
|
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_tag</span> <span class="o">==</span> <span class="s1">'oc'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Occitanis not known by babel, its closest relative is Catalan</span>
|
||||||
|
<span class="c1"># but 'ca' is already in the list of engine_traits.languages --></span>
|
||||||
|
<span class="c1"># 'oc' will be ignored.</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: </span><span class="si">%s</span><span class="s2"> is unknown by babel"</span> <span class="o">%</span> <span class="n">eng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">'zh_Hans'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'zh'</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="s1">'zh_Hant'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'zh'</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
482
_modules/searx/engines/qwant.html
Normal file
@@ -0,0 +1,482 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.qwant — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.qwant</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.qwant</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This engine uses the Qwant API (https://api.qwant.com/v3) to implement Qwant</span>
|
||||||
|
<span class="sd">-Web, -News, -Images and -Videos. The API is undocumented but can be reverse</span>
|
||||||
|
<span class="sd">engineered by reading the network log of https://www.qwant.com/ queries.</span>
|
||||||
|
|
||||||
|
<span class="sd">For Qwant's *web-search* two alternatives are implemented:</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``web``: uses the :py:obj:`api_url` which returns a JSON structure</span>
|
||||||
|
<span class="sd">- ``web-lite``: uses the :py:obj:`web_lite_url` which returns a HTML page</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following additional settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`qwant_categ`</span>
|
||||||
|
|
||||||
|
<span class="sd">This implementation is used by different qwant engines in the :ref:`settings.yml</span>
|
||||||
|
<span class="sd"><settings engines>`:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: qwant</span>
|
||||||
|
<span class="sd"> qwant_categ: web-lite # alternatively use 'web'</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> - name: qwant news</span>
|
||||||
|
<span class="sd"> qwant_categ: news</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> - name: qwant images</span>
|
||||||
|
<span class="sd"> qwant_categ: images</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> - name: qwant videos</span>
|
||||||
|
<span class="sd"> qwant_categ: videos</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">datetime</span><span class="p">,</span>
|
||||||
|
<span class="n">timedelta</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">SearxEngineAPIException</span><span class="p">,</span>
|
||||||
|
<span class="n">SearxEngineTooManyRequestsException</span><span class="p">,</span>
|
||||||
|
<span class="n">SearxEngineCaptchaException</span><span class="p">,</span>
|
||||||
|
<span class="n">SearxEngineAccessDeniedException</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">,</span>
|
||||||
|
<span class="n">get_embeded_stream_url</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.qwant.com/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q14657870'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">5</span>
|
||||||
|
<span class="sd">"""5 pages maximum (``&p=5``): Trying to do more just results in an improper</span>
|
||||||
|
<span class="sd">redirect"""</span>
|
||||||
|
|
||||||
|
<span class="c1"># Otherwise Qwant will return 403 if not set</span>
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">qwant_categ</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">"""One of ``web-lite`` (or ``web``), ``news``, ``images`` or ``videos``"""</span>
|
||||||
|
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="c1"># safe_search_map = {0: '&safesearch=0', 1: '&safesearch=1', 2: '&safesearch=2'}</span>
|
||||||
|
|
||||||
|
<span class="c1"># fmt: off</span>
|
||||||
|
<span class="n">qwant_news_locales</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'ca_ad'</span><span class="p">,</span> <span class="s1">'ca_es'</span><span class="p">,</span> <span class="s1">'ca_fr'</span><span class="p">,</span> <span class="s1">'co_fr'</span><span class="p">,</span> <span class="s1">'de_at'</span><span class="p">,</span> <span class="s1">'de_ch'</span><span class="p">,</span> <span class="s1">'de_de'</span><span class="p">,</span> <span class="s1">'en_au'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'en_ca'</span><span class="p">,</span> <span class="s1">'en_gb'</span><span class="p">,</span> <span class="s1">'en_ie'</span><span class="p">,</span> <span class="s1">'en_my'</span><span class="p">,</span> <span class="s1">'en_nz'</span><span class="p">,</span> <span class="s1">'en_us'</span><span class="p">,</span> <span class="s1">'es_ad'</span><span class="p">,</span> <span class="s1">'es_ar'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'es_cl'</span><span class="p">,</span> <span class="s1">'es_co'</span><span class="p">,</span> <span class="s1">'es_es'</span><span class="p">,</span> <span class="s1">'es_mx'</span><span class="p">,</span> <span class="s1">'es_pe'</span><span class="p">,</span> <span class="s1">'eu_es'</span><span class="p">,</span> <span class="s1">'eu_fr'</span><span class="p">,</span> <span class="s1">'fc_ca'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'fr_ad'</span><span class="p">,</span> <span class="s1">'fr_be'</span><span class="p">,</span> <span class="s1">'fr_ca'</span><span class="p">,</span> <span class="s1">'fr_ch'</span><span class="p">,</span> <span class="s1">'fr_fr'</span><span class="p">,</span> <span class="s1">'it_ch'</span><span class="p">,</span> <span class="s1">'it_it'</span><span class="p">,</span> <span class="s1">'nl_be'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'nl_nl'</span><span class="p">,</span> <span class="s1">'pt_ad'</span><span class="p">,</span> <span class="s1">'pt_pt'</span><span class="p">,</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="c1"># fmt: on</span>
|
||||||
|
|
||||||
|
<span class="c1"># search-url</span>
|
||||||
|
|
||||||
|
<span class="n">api_url</span> <span class="o">=</span> <span class="s1">'https://api.qwant.com/v3/search/'</span>
|
||||||
|
<span class="sd">"""URL of Qwant's API (JSON)"""</span>
|
||||||
|
|
||||||
|
<span class="n">web_lite_url</span> <span class="o">=</span> <span class="s1">'https://lite.qwant.com/'</span>
|
||||||
|
<span class="sd">"""URL of Qwant-Lite (HTML)"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/qwant.html#searx.engines.qwant.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Qwant search request"""</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">q_locale</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"searxng_locale"</span><span class="p">],</span> <span class="n">default</span><span class="o">=</span><span class="s1">'en_US'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">api_url</span> <span class="o">+</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">qwant_categ</span><span class="si">}</span><span class="s1">?'</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">'web-lite'</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">web_lite_url</span> <span class="o">+</span> <span class="s1">'?'</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'locale'</span><span class="p">]</span> <span class="o">=</span> <span class="n">q_locale</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'l'</span><span class="p">]</span> <span class="o">=</span> <span class="n">q_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'s'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'p'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">'images'</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'count'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">50</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'locale'</span><span class="p">]</span> <span class="o">=</span> <span class="n">q_locale</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'tgp'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'offset'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">args</span><span class="p">[</span><span class="s1">'count'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span> <span class="c1"># web, news, videos</span>
|
||||||
|
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'count'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">10</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'locale'</span><span class="p">]</span> <span class="o">=</span> <span class="n">q_locale</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'llm'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'false'</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'tgp'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'offset'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">args</span><span class="p">[</span><span class="s1">'count'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">url</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">'web-lite'</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">parse_web_lite</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">parse_web_api</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="parse_web_lite">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/qwant.html#searx.engines.qwant.parse_web_lite">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_web_lite</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse results from Qwant-Lite"""</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//section/article'</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">"./span[contains(@class, 'tooltip')]"</span><span class="p">):</span>
|
||||||
|
<span class="c1"># ignore randomly interspersed advertising adds</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s2">"./span[contains(@class, 'url partner')]"</span><span class="p">)),</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'./h2/a'</span><span class="p">)),</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'./p'</span><span class="p">)),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="parse_web_api">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/qwant.html#searx.engines.qwant.parse_web_api">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_web_api</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse results from Qwant's API"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-locals, too-many-branches, too-many-statements</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Try to load JSON result</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">search_results</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||||
|
<span class="n">search_results</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">search_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'data'</span><span class="p">,</span> <span class="p">{})</span>
|
||||||
|
|
||||||
|
<span class="c1"># check for an API error</span>
|
||||||
|
<span class="k">if</span> <span class="n">search_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'status'</span><span class="p">)</span> <span class="o">!=</span> <span class="s1">'success'</span><span class="p">:</span>
|
||||||
|
<span class="n">error_code</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'error_code'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">error_code</span> <span class="o">==</span> <span class="mi">24</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineTooManyRequestsException</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">search_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"data"</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"error_data"</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"captchaUrl"</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">403</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineAccessDeniedException</span><span class="p">()</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'message'</span><span class="p">,</span> <span class="p">[</span><span class="s1">'unknown'</span><span class="p">]))</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">msg</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">error_code</span><span class="si">}</span><span class="s2">)"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># raise for other errors</span>
|
||||||
|
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">'web'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># The WEB query contains a list named 'mainline'. This list can contain</span>
|
||||||
|
<span class="c1"># different result types (e.g. mainline[0]['type'] returns type of the</span>
|
||||||
|
<span class="c1"># result items in mainline[0]['items']</span>
|
||||||
|
<span class="n">mainline</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'result'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'items'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mainline'</span><span class="p">,</span> <span class="p">{})</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Queries on News, Images and Videos do not have a list named 'mainline'</span>
|
||||||
|
<span class="c1"># in the response. The result items are directly in the list</span>
|
||||||
|
<span class="c1"># result['items'].</span>
|
||||||
|
<span class="n">mainline</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'result'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'items'</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="n">mainline</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="p">{</span><span class="s1">'type'</span><span class="p">:</span> <span class="n">qwant_categ</span><span class="p">,</span> <span class="s1">'items'</span><span class="p">:</span> <span class="n">mainline</span><span class="p">},</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># return empty array if there are no results</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">mainline</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">mainline</span><span class="p">:</span>
|
||||||
|
<span class="n">mainline_type</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'type'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">mainline_type</span> <span class="o">!=</span> <span class="n">qwant_categ</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">'ads'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># ignore adds</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">mainline_items</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'items'</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">mainline_items</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'title'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">res_url</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'url'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">'web'</span><span class="p">:</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'desc'</span><span class="p">]</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">res_url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">'news'</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'date'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">pub_date</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">pub_date</span><span class="p">)</span>
|
||||||
|
<span class="n">news_media</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'media'</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">news_media</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">news_media</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'pict'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'url'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">res_url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">pub_date</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">'images'</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'thumbnail'</span><span class="p">]</span>
|
||||||
|
<span class="n">img_src</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'media'</span><span class="p">]</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">res_url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'images.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail_src'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">img_src</span><span class="p">,</span>
|
||||||
|
<span class="s1">'resolution'</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s1">'width'</span><span class="p">]</span><span class="si">}</span><span class="s2"> x </span><span class="si">{</span><span class="n">item</span><span class="p">[</span><span class="s1">'height'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="s1">'img_format'</span><span class="p">:</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'thumb_type'</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">mainline_type</span> <span class="o">==</span> <span class="s1">'videos'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># some videos do not have a description: while qwant-video</span>
|
||||||
|
<span class="c1"># returns an empty string, such video from a qwant-web query</span>
|
||||||
|
<span class="c1"># miss the 'desc' key.</span>
|
||||||
|
<span class="n">d</span><span class="p">,</span> <span class="n">s</span><span class="p">,</span> <span class="n">c</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'desc'</span><span class="p">),</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'source'</span><span class="p">),</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'channel'</span><span class="p">)</span>
|
||||||
|
<span class="n">content_parts</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">if</span> <span class="n">d</span><span class="p">:</span>
|
||||||
|
<span class="n">content_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">s</span><span class="p">:</span>
|
||||||
|
<span class="n">content_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2"> "</span> <span class="o">%</span> <span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Source"</span><span class="p">),</span> <span class="n">s</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">c</span><span class="p">:</span>
|
||||||
|
<span class="n">content_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2"> "</span> <span class="o">%</span> <span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Channel"</span><span class="p">),</span> <span class="n">c</span><span class="p">))</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="s1">' // '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">content_parts</span><span class="p">)</span>
|
||||||
|
<span class="n">length</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'duration'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">length</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">length</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">milliseconds</span><span class="o">=</span><span class="n">length</span><span class="p">)</span>
|
||||||
|
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'date'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">pub_date</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">pub_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">pub_date</span><span class="p">)</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s1">'thumbnail'</span><span class="p">]</span>
|
||||||
|
<span class="c1"># from some locations (DE and others?) the s2 link do</span>
|
||||||
|
<span class="c1"># response a 'Please wait ..' but does not deliver the thumbnail</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'https://s2.qwant.com'</span><span class="p">,</span> <span class="s1">'https://s1.qwant.com'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">res_url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'iframe_src'</span><span class="p">:</span> <span class="n">get_embeded_stream_url</span><span class="p">(</span><span class="n">res_url</span><span class="p">),</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">pub_date</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'videos.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'length'</span><span class="p">:</span> <span class="n">length</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">network</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">region_tag</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extr</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">about</span><span class="p">[</span><span class="s1">'website'</span><span class="p">])</span>
|
||||||
|
<span class="n">json_string</span> <span class="o">=</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="s1">'INITIAL_PROPS = '</span><span class="p">,</span> <span class="s1">'</script>'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">q_initial_props</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">json_string</span><span class="p">)</span>
|
||||||
|
<span class="n">q_locales</span> <span class="o">=</span> <span class="n">q_initial_props</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'locales'</span><span class="p">)</span>
|
||||||
|
<span class="n">eng_tag_list</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">country</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">q_locales</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">v</span><span class="p">[</span><span class="s1">'langs'</span><span class="p">]:</span>
|
||||||
|
<span class="n">_locale</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{lang}</span><span class="s2">_</span><span class="si">{country}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">lang</span><span class="o">=</span><span class="n">lang</span><span class="p">,</span> <span class="n">country</span><span class="o">=</span><span class="n">country</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">qwant_categ</span> <span class="o">==</span> <span class="s1">'news'</span> <span class="ow">and</span> <span class="n">_locale</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">qwant_news_locales</span><span class="p">:</span>
|
||||||
|
<span class="c1"># qwant-news does not support all locales from qwant-web:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">eng_tag_list</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">_locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="n">eng_tag_list</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'_'</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: can't determine babel locale of quant's locale </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">eng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
329
_modules/searx/engines/radio_browser.html
Normal file
@@ -0,0 +1,329 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.radio_browser — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.radio_browser</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.radio_browser</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Search radio stations from RadioBrowser by `Advanced station search API`_.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Advanced station search API:</span>
|
||||||
|
<span class="sd"> https://de1.api.radio-browser.info/#Advanced_station_search</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">random</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">socket</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.radio-browser.info/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q111664849'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://de1.api.radio-browser.info/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'music'</span><span class="p">,</span> <span class="s1">'radio'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">number_of_results</span> <span class="o">=</span> <span class="mi">10</span>
|
||||||
|
|
||||||
|
<span class="n">station_filters</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># ['countrycode', 'language']</span>
|
||||||
|
<span class="sd">"""A list of filters to be applied to the search of radio stations. By default</span>
|
||||||
|
<span class="sd">none filters are applied. Valid filters are:</span>
|
||||||
|
|
||||||
|
<span class="sd">``language``</span>
|
||||||
|
<span class="sd"> Filter stations by selected language. For instance the ``de`` from ``:de-AU``</span>
|
||||||
|
<span class="sd"> will be translated to `german` and used in the argument ``language=``.</span>
|
||||||
|
|
||||||
|
<span class="sd">``countrycode``</span>
|
||||||
|
<span class="sd"> Filter stations by selected country. The 2-digit countrycode of the station</span>
|
||||||
|
<span class="sd"> comes from the region the user selected. For instance ``:de-AU`` will filter</span>
|
||||||
|
<span class="sd"> out all stations not in ``AU``.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> RadioBrowser has registered a lot of languages and countrycodes unknown to</span>
|
||||||
|
<span class="sd"> :py:obj:`babel` and note that when searching for radio stations, users are</span>
|
||||||
|
<span class="sd"> more likely to search by name than by region or language.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">CACHE</span><span class="p">:</span> <span class="n">EngineCache</span>
|
||||||
|
<span class="sd">"""Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
|
||||||
|
<span class="sd">seconds."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">_</span><span class="p">):</span>
|
||||||
|
<span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="s2">"radio_browser"</span><span class="p">)</span>
|
||||||
|
<span class="n">server_list</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">server_list</span><span class="p">()</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="n">servers</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"servers"</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="k">if</span> <span class="n">servers</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">servers</span>
|
||||||
|
|
||||||
|
<span class="c1"># hint: can take up to 40sec!</span>
|
||||||
|
<span class="n">ips</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">getaddrinfo</span><span class="p">(</span><span class="s2">"all.api.radio-browser.info"</span><span class="p">,</span> <span class="mi">80</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">socket</span><span class="o">.</span><span class="n">IPPROTO_TCP</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">ip_tuple</span> <span class="ow">in</span> <span class="n">ips</span><span class="p">:</span>
|
||||||
|
<span class="n">_ip</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">ip_tuple</span><span class="p">[</span><span class="mi">4</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">gethostbyaddr</span><span class="p">(</span><span class="n">_ip</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">srv</span> <span class="o">=</span> <span class="s2">"https://"</span> <span class="o">+</span> <span class="n">url</span>
|
||||||
|
<span class="k">if</span> <span class="n">srv</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">servers</span><span class="p">:</span>
|
||||||
|
<span class="n">servers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">srv</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># update server list once in 24h</span>
|
||||||
|
<span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s2">"servers"</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">servers</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">servers</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">servers</span> <span class="o">=</span> <span class="n">server_list</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">servers</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Fetched server list is empty!"</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="n">server</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">servers</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'name'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'order'</span><span class="p">:</span> <span class="s1">'votes'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'offset'</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">number_of_results</span><span class="p">,</span>
|
||||||
|
<span class="s1">'limit'</span><span class="p">:</span> <span class="n">number_of_results</span><span class="p">,</span>
|
||||||
|
<span class="s1">'hidebroken'</span><span class="p">:</span> <span class="s1">'true'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'reverse'</span><span class="p">:</span> <span class="s1">'true'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'language'</span> <span class="ow">in</span> <span class="n">station_filters</span><span class="p">:</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">])</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">if</span> <span class="n">lang</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'countrycode'</span> <span class="ow">in</span> <span class="n">station_filters</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">))</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">countrycode</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">countrycode</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'countrycodes'</span><span class="p">]:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'countrycode'</span><span class="p">]</span> <span class="o">=</span> <span class="n">countrycode</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">server</span><span class="si">}</span><span class="s2">/json/stations/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">json_resp</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_resp</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'homepage'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">url</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'url_resolved'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">tags</span> <span class="o">=</span> <span class="s1">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tags'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">tags</span><span class="p">:</span>
|
||||||
|
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tags</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'state'</span><span class="p">,</span> <span class="s1">'country'</span><span class="p">]:</span>
|
||||||
|
<span class="n">v</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">v</span><span class="p">:</span>
|
||||||
|
<span class="n">v</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">metadata</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">codec</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'codec'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">codec</span> <span class="ow">and</span> <span class="n">codec</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="s1">'unknown'</span><span class="p">:</span>
|
||||||
|
<span class="n">metadata</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">codec</span><span class="si">}</span><span class="s1"> '</span> <span class="o">+</span> <span class="n">gettext</span><span class="p">(</span><span class="s1">'radio'</span><span class="p">))</span>
|
||||||
|
<span class="k">for</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s1">'bitrate'</span><span class="p">),</span> <span class="s1">'bitrate'</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s1">'votes'</span><span class="p">),</span> <span class="s1">'votes'</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="n">gettext</span><span class="p">(</span><span class="s1">'clicks'</span><span class="p">),</span> <span class="s1">'clickcount'</span><span class="p">),</span>
|
||||||
|
<span class="p">]:</span>
|
||||||
|
<span class="n">v</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">v</span><span class="p">:</span>
|
||||||
|
<span class="n">v</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="n">metadata</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">x</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'favicon'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"http://"</span><span class="p">,</span> <span class="s2">"https://"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="s1">' | '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">content</span><span class="p">),</span>
|
||||||
|
<span class="s1">'metadata'</span><span class="p">:</span> <span class="s1">' | '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata</span><span class="p">),</span>
|
||||||
|
<span class="s1">'iframe_src'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'url_resolved'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"http://"</span><span class="p">,</span> <span class="s2">"https://"</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/radio_browser.html#searx.engines.radio_browser.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages and countrycodes from RadioBrowser</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``traits.languages``: `list of languages API`_</span>
|
||||||
|
<span class="sd"> - ``traits.custom['countrycodes']``: `list of countries API`_</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _list of countries API: https://de1.api.radio-browser.info/#List_of_countries</span>
|
||||||
|
<span class="sd"> .. _list of languages API: https://de1.api.radio-browser.info/#List_of_languages</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||||
|
|
||||||
|
<span class="n">init</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">babel.core</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_global</span>
|
||||||
|
|
||||||
|
<span class="n">babel_reg_list</span> <span class="o">=</span> <span class="n">get_global</span><span class="p">(</span><span class="s2">"territory_languages"</span><span class="p">)</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">server</span> <span class="o">=</span> <span class="n">server_list</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">language_list</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">server</span><span class="si">}</span><span class="s1">/json/languages'</span><span class="p">)</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">country_list</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">server</span><span class="si">}</span><span class="s1">/json/countries'</span><span class="p">)</span><span class="o">.</span><span class="n">json</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">language_list</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">babel_lang</span> <span class="o">=</span> <span class="n">lang</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'iso_639'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">babel_lang</span><span class="p">:</span>
|
||||||
|
<span class="c1"># the language doesn't have any iso code, and hence can't be parsed</span>
|
||||||
|
<span class="c1"># print(f"ERROR: lang - no iso code in {lang}")</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">babel_lang</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s2">"-"</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="c1"># print(f"ERROR: language tag {babel_lang} is unknown by babel")</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">lang</span><span class="p">[</span><span class="s1">'name'</span><span class="p">]</span>
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
|
||||||
|
<span class="n">countrycodes</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">region</span> <span class="ow">in</span> <span class="n">country_list</span><span class="p">:</span>
|
||||||
|
<span class="c1"># country_list contains duplicates that differ only in upper/lower case</span>
|
||||||
|
<span class="n">_reg</span> <span class="o">=</span> <span class="n">region</span><span class="p">[</span><span class="s1">'iso_3166_1'</span><span class="p">]</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">_reg</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">babel_reg_list</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"ERROR: region tag </span><span class="si">{</span><span class="n">region</span><span class="p">[</span><span class="s1">'iso_3166_1'</span><span class="p">]</span><span class="si">}</span><span class="s2"> is unknown by babel"</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">countrycodes</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">_reg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">countrycodes</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">countrycodes</span><span class="p">)</span>
|
||||||
|
<span class="n">countrycodes</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'countrycodes'</span><span class="p">]</span> <span class="o">=</span> <span class="n">countrycodes</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
267
_modules/searx/engines/recoll.html
Normal file
@@ -0,0 +1,267 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.recoll — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.recoll</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.recoll</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">""".. sidebar:: info</span>
|
||||||
|
|
||||||
|
<span class="sd"> - `Recoll <https://www.lesbonscomptes.com/recoll/>`_</span>
|
||||||
|
<span class="sd"> - `recoll-webui <https://framagit.org/medoc92/recollwebui.git>`_</span>
|
||||||
|
<span class="sd"> - :origin:`searx/engines/recoll.py`</span>
|
||||||
|
|
||||||
|
<span class="sd">Recoll_ is a desktop full-text search tool based on Xapian. By itself Recoll_</span>
|
||||||
|
<span class="sd">does not offer WEB or API access, this can be achieved using recoll-webui_</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">You must configure the following settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`base_url`</span>
|
||||||
|
<span class="sd">- :py:obj:`mount_prefix`</span>
|
||||||
|
<span class="sd">- :py:obj:`dl_prefix`</span>
|
||||||
|
<span class="sd">- :py:obj:`search_dir`</span>
|
||||||
|
|
||||||
|
<span class="sd">Example scenario:</span>
|
||||||
|
|
||||||
|
<span class="sd">#. Recoll indexes a local filesystem mounted in ``/export/documents/reference``,</span>
|
||||||
|
<span class="sd">#. the Recoll search interface can be reached at https://recoll.example.org/ and</span>
|
||||||
|
<span class="sd">#. the contents of this filesystem can be reached though https://download.example.org/reference</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> base_url: https://recoll.example.org</span>
|
||||||
|
<span class="sd"> mount_prefix: /export/documents</span>
|
||||||
|
<span class="sd"> dl_prefix: https://download.example.org</span>
|
||||||
|
<span class="sd"> search_dir: ""</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">date</span><span class="p">,</span> <span class="n">timedelta</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q15735774"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"https://www.lesbonscomptes.com/recoll/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Location where recoll-webui can be reached."""</span>
|
||||||
|
|
||||||
|
<span class="n">mount_prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Location where the file hierarchy is mounted on your *local* filesystem."""</span>
|
||||||
|
|
||||||
|
<span class="n">dl_prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Location where the file hierarchy as indexed by recoll can be reached."""</span>
|
||||||
|
|
||||||
|
<span class="n">search_dir</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Part of the indexed file hierarchy to be search, if empty the full domain is</span>
|
||||||
|
<span class="sd">searched."""</span>
|
||||||
|
|
||||||
|
<span class="n">_s2i</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"day"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">"week"</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> <span class="s2">"month"</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span> <span class="s2">"year"</span><span class="p">:</span> <span class="mi">365</span><span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/recoll.html#searx.engines.recoll.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the Recoll engine, checks if the mandatory values are</span>
|
||||||
|
<span class="sd"> configured.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">missing</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">cfg_name</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"base_url"</span><span class="p">,</span> <span class="s2">"mount_prefix"</span><span class="p">,</span> <span class="s2">"dl_prefix"</span><span class="p">]:</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cfg_name</span><span class="p">):</span>
|
||||||
|
<span class="n">missing</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">cfg_name</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">missing</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"missing recoll configuration: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">missing</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s2">"base_url"</span><span class="p">]</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">"/"</span><span class="p">):</span>
|
||||||
|
<span class="n">engine_settings</span><span class="p">[</span><span class="s2">"base_url"</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s2">"base_url"</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">search_after</span><span class="p">(</span><span class="n">time_range</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="n">offset</span> <span class="o">=</span> <span class="n">_s2i</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">time_range</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">offset</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span>
|
||||||
|
<span class="k">return</span> <span class="p">(</span><span class="n">date</span><span class="o">.</span><span class="n">today</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="n">offset</span><span class="p">))</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"query"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"page"</span><span class="p">:</span> <span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">],</span>
|
||||||
|
<span class="s2">"after"</span><span class="p">:</span> <span class="n">search_after</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"time_range"</span><span class="p">]),</span>
|
||||||
|
<span class="s2">"dir"</span><span class="p">:</span> <span class="n">search_dir</span><span class="p">,</span>
|
||||||
|
<span class="s2">"highlight"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/json?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">json_data</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"results"</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"url"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"file://"</span> <span class="o">+</span> <span class="n">mount_prefix</span><span class="p">,</span> <span class="n">dl_prefix</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">mtype</span> <span class="o">=</span> <span class="n">subtype</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"mtype"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">mtype</span><span class="p">:</span>
|
||||||
|
<span class="n">mtype</span><span class="p">,</span> <span class="n">subtype</span> <span class="o">=</span> <span class="p">(</span><span class="n">mtype</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"/"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="p">[</span><span class="s2">""</span><span class="p">])[:</span><span class="mi">2</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># facilitate preview support for known mime types</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">embedded</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"audio"</span><span class="p">,</span> <span class="s2">"video"</span><span class="p">]:</span>
|
||||||
|
<span class="n">embedded</span> <span class="o">=</span> <span class="n">url</span>
|
||||||
|
<span class="k">if</span> <span class="n">mtype</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"image"</span><span class="p">]</span> <span class="ow">and</span> <span class="n">subtype</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"bmp"</span><span class="p">,</span> <span class="s2">"gif"</span><span class="p">,</span> <span class="s2">"jpeg"</span><span class="p">,</span> <span class="s2">"png"</span><span class="p">]:</span>
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">url</span>
|
||||||
|
|
||||||
|
<span class="c1"># remove HTML from snippet</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"snippet"</span><span class="p">,</span> <span class="s2">""</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">File</span><span class="p">(</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"label"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="n">size</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"size"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span>
|
||||||
|
<span class="n">filename</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"filename"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span>
|
||||||
|
<span class="n">abstract</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"abstract"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span>
|
||||||
|
<span class="n">author</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"author"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span>
|
||||||
|
<span class="n">mtype</span><span class="o">=</span><span class="n">mtype</span><span class="p">,</span>
|
||||||
|
<span class="n">subtype</span><span class="o">=</span><span class="n">subtype</span><span class="p">,</span>
|
||||||
|
<span class="n">time</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"time"</span><span class="p">,</span> <span class="s2">""</span><span class="p">),</span>
|
||||||
|
<span class="n">embedded</span><span class="o">=</span><span class="n">embedded</span><span class="p">,</span>
|
||||||
|
<span class="n">thumbnail</span><span class="o">=</span><span class="n">thumbnail</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
248
_modules/searx/engines/reuters.html
Normal file
@@ -0,0 +1,248 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.reuters — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.reuters</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.reuters</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Reuters_ (news) is an international news agency.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Reuters: https://www.reuters.com</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following additional settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`sort_order`</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: reuters</span>
|
||||||
|
<span class="sd"> engine: reuters</span>
|
||||||
|
<span class="sd"> shortcut: reu</span>
|
||||||
|
<span class="sd"> sort_order: "relevance"</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">dumps</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote_plus</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil</span><span class="w"> </span><span class="kn">import</span> <span class="n">parser</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://www.reuters.com"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q130879"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"news"</span><span class="p">]</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://www.reuters.com"</span>
|
||||||
|
|
||||||
|
<span class="n">results_per_page</span> <span class="o">=</span> <span class="mi">20</span>
|
||||||
|
<span class="n">sort_order</span> <span class="o">=</span> <span class="s2">"relevance"</span>
|
||||||
|
<span class="sd">"""Sort order, one of ``relevance``, ``display_date:desc`` or ``display_data:asc``."""</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_duration_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"day"</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="s2">"week"</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span>
|
||||||
|
<span class="s2">"month"</span><span class="p">:</span> <span class="mi">30</span><span class="p">,</span>
|
||||||
|
<span class="s2">"year"</span><span class="p">:</span> <span class="mi">365</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"keyword"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"offset"</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">results_per_page</span><span class="p">,</span>
|
||||||
|
<span class="s2">"orderby"</span><span class="p">:</span> <span class="n">sort_order</span><span class="p">,</span>
|
||||||
|
<span class="s2">"size"</span><span class="p">:</span> <span class="n">results_per_page</span><span class="p">,</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"reuters"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s2">"time_range"</span><span class="p">]:</span>
|
||||||
|
<span class="n">time_diff_days</span> <span class="o">=</span> <span class="n">time_range_duration_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s2">"time_range"</span><span class="p">]]</span>
|
||||||
|
<span class="n">start_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="n">time_diff_days</span><span class="p">)</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s2">"start_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">start_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/pf/api/v3/content/fetch/articles-by-search-v2?query=</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">dumps</span><span class="p">(</span><span class="n">args</span><span class="p">))</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">resp_json</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"result"</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">resp_json</span><span class="p">[</span><span class="s2">"result"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"articles"</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">base_url</span> <span class="o">+</span> <span class="n">result</span><span class="p">[</span><span class="s2">"canonical_url"</span><span class="p">],</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"web"</span><span class="p">],</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">result</span><span class="p">[</span><span class="s2">"description"</span><span class="p">],</span>
|
||||||
|
<span class="n">thumbnail</span><span class="o">=</span><span class="n">resize_url</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"thumbnail"</span><span class="p">,</span> <span class="p">{}),</span> <span class="n">height</span><span class="o">=</span><span class="mi">80</span><span class="p">),</span>
|
||||||
|
<span class="n">metadata</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"kicker"</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"name"</span><span class="p">),</span>
|
||||||
|
<span class="n">publishedDate</span><span class="o">=</span><span class="n">parser</span><span class="o">.</span><span class="n">isoparse</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s2">"display_time"</span><span class="p">]),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="resize_url">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/reuters.html#searx.engines.reuters.resize_url">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">resize_url</span><span class="p">(</span><span class="n">thumbnail</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">width</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="n">height</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Generates a URL for Reuter's thumbnail with the dimensions *width* and</span>
|
||||||
|
<span class="sd"> *height*. If no URL can be generated from the *thumbnail data*, an empty</span>
|
||||||
|
<span class="sd"> string will be returned.</span>
|
||||||
|
|
||||||
|
<span class="sd"> width: default is *unset* (``0``)</span>
|
||||||
|
<span class="sd"> Image width in pixels (negative values are ignored). If only width is</span>
|
||||||
|
<span class="sd"> specified, the height matches the original aspect ratio.</span>
|
||||||
|
|
||||||
|
<span class="sd"> height: default is *unset* (``0``)</span>
|
||||||
|
<span class="sd"> Image height in pixels (negative values are ignored). If only height is</span>
|
||||||
|
<span class="sd"> specified, the width matches the original aspect ratio.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The file size of a full-size image is usually several MB; when reduced to a</span>
|
||||||
|
<span class="sd"> height of, for example, 80 points, only a few KB remain!</span>
|
||||||
|
|
||||||
|
<span class="sd"> Fields of the *thumbnail data* (``result.articles.[<int>].thumbnail``):</span>
|
||||||
|
|
||||||
|
<span class="sd"> thumbnail.url:</span>
|
||||||
|
<span class="sd"> Is a full-size image (>MB).</span>
|
||||||
|
|
||||||
|
<span class="sd"> thumbnail.width & .height:</span>
|
||||||
|
<span class="sd"> Dimensions of the full-size image.</span>
|
||||||
|
|
||||||
|
<span class="sd"> thumbnail.resizer_url:</span>
|
||||||
|
<span class="sd"> Reuters has a *resizer* `REST-API for the images`_, this is the URL of the</span>
|
||||||
|
<span class="sd"> service. This URL includes the ``&auth`` argument, other arguments are</span>
|
||||||
|
<span class="sd"> ``&width=<int>`` and ``&height=<int>``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _REST-API for the images:</span>
|
||||||
|
<span class="sd"> https://dev.arcxp.com/photo-center/image-resizer/resizer-v2-how-to-transform-images/#query-parameters</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"resizer_url"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">url</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span>
|
||||||
|
<span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">width</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">+=</span> <span class="sa">f</span><span class="s2">"&width=</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">width</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">height</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">+=</span> <span class="sa">f</span><span class="s2">"&height=</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="n">height</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">return</span> <span class="n">url</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
186
_modules/searx/engines/sepiasearch.html
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.sepiasearch — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.sepiasearch</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.sepiasearch</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""SepiaSearch uses the same languages as :py:obj:`Peertube</span>
|
||||||
|
<span class="sd"><searx.engines.peertube>` and the response is identical to the response from the</span>
|
||||||
|
<span class="sd">peertube engines.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.peertube</span><span class="w"> </span><span class="kn">import</span> <span class="n">fetch_traits</span> <span class="c1"># pylint: disable=unused-import</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.peertube</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="c1"># pylint: disable=unused-import</span>
|
||||||
|
<span class="n">video_response</span><span class="p">,</span>
|
||||||
|
<span class="n">safesearch_table</span><span class="p">,</span>
|
||||||
|
<span class="n">time_range_table</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="c1"># pylint: disable=line-too-long</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://sepiasearch.org'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://docs.joinpeertube.org/api-rest-reference.html#tag/Search/operation/searchVideos'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'videos'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://sepiasearch.org'</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/peertube.html#searx.engines.sepiasearch.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Assemble request for the SepiaSearch API"""</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="c1"># eng_region = traits.get_region(params['searxng_locale'], 'en_US')</span>
|
||||||
|
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"/api/v1/search/videos?"</span>
|
||||||
|
<span class="o">+</span> <span class="n">urlencode</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'search'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'start'</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="mi">10</span><span class="p">,</span>
|
||||||
|
<span class="s1">'count'</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
|
||||||
|
<span class="c1"># -createdAt: sort by date ascending / createdAt: date descending</span>
|
||||||
|
<span class="s1">'sort'</span><span class="p">:</span> <span class="s1">'-match'</span><span class="p">,</span> <span class="c1"># sort by *match descending*</span>
|
||||||
|
<span class="s1">'nsfw'</span><span class="p">:</span> <span class="n">safesearch_table</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">eng_lang</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">'&languageOneOf[]='</span> <span class="o">+</span> <span class="n">eng_lang</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">'&boostLanguages[]='</span> <span class="o">+</span> <span class="n">eng_lang</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]</span> <span class="ow">in</span> <span class="n">time_range_table</span><span class="p">:</span>
|
||||||
|
<span class="n">time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">date</span><span class="p">()</span> <span class="o">+</span> <span class="n">time_range_table</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">]]</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">+=</span> <span class="s1">'&startDate='</span> <span class="o">+</span> <span class="n">time</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
286
_modules/searx/engines/springer.html
Normal file
@@ -0,0 +1,286 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.springer — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.springer</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.springer</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""`Springer Nature`_ is a global publisher dedicated to providing service to</span>
|
||||||
|
<span class="sd">research community with official Springer-API_ (API-Playground_).</span>
|
||||||
|
|
||||||
|
<span class="sd">.. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> The Springer engine requires an API key, which can be obtained via the</span>
|
||||||
|
<span class="sd"> `Springer subscription`_.</span>
|
||||||
|
|
||||||
|
<span class="sd">Since the search term is passed 1:1 to the API, SearXNG users can use the</span>
|
||||||
|
<span class="sd">`Supported Query Parameters`_.</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``!springer (doi:10.1007/s10948-025-07019-1 OR doi:10.1007/s10948-025-07035-1)``</span>
|
||||||
|
<span class="sd">- ``!springer keyword:ybco``</span>
|
||||||
|
|
||||||
|
<span class="sd">However, please note that the available options depend on the subscription type.</span>
|
||||||
|
|
||||||
|
<span class="sd">For example, the ``year:`` filter requires a *Premium Plan* subscription.</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``!springer keyword:ybco year:2024``</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine uses the REST Meta-API_ `v2` endpoint, but there is also a `Python</span>
|
||||||
|
<span class="sd">API Wrapper`_.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Python API Wrapper: https://pypi.org/project/springernature-api-client/</span>
|
||||||
|
<span class="sd">.. _Springer Nature: https://www.springernature.com/</span>
|
||||||
|
<span class="sd">.. _Springer subscription: https://dev.springernature.com/subscription/</span>
|
||||||
|
<span class="sd">.. _Springer-API: https://dev.springernature.com/docs/introduction/</span>
|
||||||
|
<span class="sd">.. _API-Playground: https://dev.springernature.com/docs/live-documentation/</span>
|
||||||
|
<span class="sd">.. _Meta-API: https://dev.springernature.com/docs/api-endpoints/meta-api/</span>
|
||||||
|
<span class="sd">.. _Supported Query Parameters: https://dev.springernature.com/docs/supported-query-params/</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following additional settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`api_key`</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: springer nature</span>
|
||||||
|
<span class="sd"> api_key: "..."</span>
|
||||||
|
<span class="sd"> inactive: false</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://www.springernature.com/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q21096327"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"https://dev.springernature.com/docs/live-documentation/"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"JSON"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"science"</span><span class="p">,</span> <span class="s2">"scientific publications"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">nb_per_page</span> <span class="o">=</span> <span class="mi">10</span>
|
||||||
|
<span class="sd">"""Number of results to return in the request, see `Pagination and Limits`_ for</span>
|
||||||
|
<span class="sd">more details.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Pagination and Limits:</span>
|
||||||
|
<span class="sd"> https://dev.springernature.com/docs/advanced-querying/pagination-limits/</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">api_key</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Key used for the Meta-API_. Get your API key from: `Springer subscription`_"""</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://api.springernature.com/meta/v2/json"</span>
|
||||||
|
<span class="sd">"""An enhanced endpoint with additional metadata fields and optimized queries</span>
|
||||||
|
<span class="sd">for more efficient and comprehensive retrieval (Meta-API_ `v2`).</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/springer.html#searx.engines.springer.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the Springer engine, checks whether the</span>
|
||||||
|
<span class="sd"> :py:obj:`api_key` is set, otherwise the engine is inactive.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">engine_settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"api_key"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Springer's API key is a hex value</span>
|
||||||
|
<span class="nb">int</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="mi">16</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Springer's API key is not set or invalid."</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"api_key"</span><span class="p">:</span> <span class="n">api_key</span><span class="p">,</span>
|
||||||
|
<span class="s2">"q"</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s2">"s"</span><span class="p">:</span> <span class="n">nb_per_page</span> <span class="o">*</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">),</span>
|
||||||
|
<span class="s2">"p"</span><span class="p">:</span> <span class="n">nb_per_page</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="c1"># For example, the ``year:`` filter requires a *Premium Plan* subscription.</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"raise_for_httperror"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="p">(</span>
|
||||||
|
<span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">403</span>
|
||||||
|
<span class="ow">and</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">"status"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"fail"</span>
|
||||||
|
<span class="ow">and</span> <span class="s2">"premium feature"</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">"message"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">field</span><span class="p">(</span><span class="n">k</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="s2">""</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||||
|
<span class="n">published</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"publicationDate"</span><span class="p">],</span> <span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">authors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">" "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">author</span><span class="p">[</span><span class="s2">"creator"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">", "</span><span class="p">)[::</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="k">for</span> <span class="n">author</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"creators"</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="n">pdf_url</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">html_url</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">url_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">url_list</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">item</span><span class="p">[</span><span class="s2">"platform"</span><span class="p">]</span> <span class="o">!=</span> <span class="s2">"web"</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">item</span><span class="p">[</span><span class="s2">"value"</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"http://"</span><span class="p">,</span> <span class="s2">"https://"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">item</span><span class="p">[</span><span class="s2">"format"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"html"</span><span class="p">:</span>
|
||||||
|
<span class="n">html_url</span> <span class="o">=</span> <span class="n">val</span>
|
||||||
|
<span class="k">elif</span> <span class="n">item</span><span class="p">[</span><span class="s2">"format"</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"pdf"</span><span class="p">:</span>
|
||||||
|
<span class="n">pdf_url</span> <span class="o">=</span> <span class="n">val</span>
|
||||||
|
|
||||||
|
<span class="n">paper</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">html_url</span><span class="p">,</span>
|
||||||
|
<span class="c1"># html_url=html_url,</span>
|
||||||
|
<span class="n">pdf_url</span><span class="o">=</span><span class="n">pdf_url</span><span class="p">,</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"title"</span><span class="p">),</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"abstract"</span><span class="p">),</span>
|
||||||
|
<span class="n">comments</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"publicationName"</span><span class="p">),</span>
|
||||||
|
<span class="n">tags</span><span class="o">=</span><span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"keyword"</span><span class="p">,</span> <span class="p">[]),</span>
|
||||||
|
<span class="n">publishedDate</span><span class="o">=</span><span class="n">published</span><span class="p">,</span>
|
||||||
|
<span class="nb">type</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"contentType"</span><span class="p">),</span>
|
||||||
|
<span class="n">authors</span><span class="o">=</span><span class="n">authors</span><span class="p">,</span>
|
||||||
|
<span class="n">publisher</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"publisher"</span><span class="p">),</span>
|
||||||
|
<span class="n">journal</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"publicationName"</span><span class="p">),</span>
|
||||||
|
<span class="n">volume</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"volume"</span><span class="p">),</span>
|
||||||
|
<span class="n">pages</span><span class="o">=</span><span class="s2">"-"</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">field</span><span class="p">(</span><span class="s2">"startingPage"</span><span class="p">),</span> <span class="n">field</span><span class="p">(</span><span class="s2">"endingPage"</span><span class="p">)]</span> <span class="k">if</span> <span class="n">x</span><span class="p">]),</span>
|
||||||
|
<span class="n">number</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"number"</span><span class="p">),</span>
|
||||||
|
<span class="n">doi</span><span class="o">=</span><span class="n">field</span><span class="p">(</span><span class="s2">"doi"</span><span class="p">),</span>
|
||||||
|
<span class="n">issn</span><span class="o">=</span><span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">field</span><span class="p">(</span><span class="s2">"issn"</span><span class="p">)]</span> <span class="k">if</span> <span class="n">x</span><span class="p">],</span>
|
||||||
|
<span class="n">isbn</span><span class="o">=</span><span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="n">field</span><span class="p">(</span><span class="s2">"isbn"</span><span class="p">)]</span> <span class="k">if</span> <span class="n">x</span><span class="p">],</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">paper</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
230
_modules/searx/engines/sqlite.html
Normal file
@@ -0,0 +1,230 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.sqlite — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.sqlite</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.sqlite</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""SQLite is a small, fast and reliable SQL database engine. It does not require</span>
|
||||||
|
<span class="sd">any extra dependency.</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following (additional) settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`result_type`</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Example</span>
|
||||||
|
<span class="sd">=======</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _MediathekView: https://mediathekview.de/</span>
|
||||||
|
|
||||||
|
<span class="sd">To demonstrate the power of database engines, here is a more complex example</span>
|
||||||
|
<span class="sd">which reads from a MediathekView_ (DE) movie database. For this example of the</span>
|
||||||
|
<span class="sd">SQLite engine download the database:</span>
|
||||||
|
|
||||||
|
<span class="sd">- https://liste.mediathekview.de/filmliste-v2.db.bz2</span>
|
||||||
|
|
||||||
|
<span class="sd">and unpack into ``searx/data/filmliste-v2.db``. To search the database use e.g</span>
|
||||||
|
<span class="sd">Query to test: ``!mediathekview concert``</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: mediathekview</span>
|
||||||
|
<span class="sd"> engine: sqlite</span>
|
||||||
|
<span class="sd"> shortcut: mediathekview</span>
|
||||||
|
<span class="sd"> categories: [general, videos]</span>
|
||||||
|
<span class="sd"> result_type: MainResult</span>
|
||||||
|
<span class="sd"> database: searx/data/filmliste-v2.db</span>
|
||||||
|
<span class="sd"> query_str: >-</span>
|
||||||
|
<span class="sd"> SELECT title || ' (' || time(duration, 'unixepoch') || ')' AS title,</span>
|
||||||
|
<span class="sd"> COALESCE( NULLIF(url_video_hd,''), NULLIF(url_video_sd,''), url_video) AS url,</span>
|
||||||
|
<span class="sd"> description AS content</span>
|
||||||
|
<span class="sd"> FROM film</span>
|
||||||
|
<span class="sd"> WHERE title LIKE :wildcard OR description LIKE :wildcard</span>
|
||||||
|
<span class="sd"> ORDER BY duration DESC</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">sqlite3</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">contextlib</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span><span class="p">,</span> <span class="n">KeyValue</span>
|
||||||
|
|
||||||
|
<span class="n">engine_type</span> <span class="o">=</span> <span class="s2">"offline"</span>
|
||||||
|
|
||||||
|
<span class="n">database</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Filename of the SQLite DB."""</span>
|
||||||
|
|
||||||
|
<span class="n">query_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""SQL query that returns the result items."""</span>
|
||||||
|
|
||||||
|
<span class="n">result_type</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"MainResult"</span><span class="p">,</span> <span class="s2">"KeyValue"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"KeyValue"</span>
|
||||||
|
<span class="sd">"""The result type can be :py:obj:`MainResult` or :py:obj:`KeyValue`."""</span>
|
||||||
|
|
||||||
|
<span class="n">limit</span> <span class="o">=</span> <span class="mi">10</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="s1">'query_str'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">engine_settings</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'query_str cannot be empty'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">engine_settings</span><span class="p">[</span><span class="s1">'query_str'</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'select '</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'only SELECT query is supported'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="sqlite_cursor">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/offline/sql-engines.html#searx.engines.sqlite.sqlite_cursor">[docs]</a>
|
||||||
|
<span class="nd">@contextlib</span><span class="o">.</span><span class="n">contextmanager</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">sqlite_cursor</span><span class="p">():</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Implements a :py:obj:`Context Manager <contextlib.contextmanager>` for a</span>
|
||||||
|
<span class="sd"> :py:obj:`sqlite3.Cursor`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Open database in read only mode: if the database doesn't exist. The default</span>
|
||||||
|
<span class="sd"> mode creates an empty file on the file system. See:</span>
|
||||||
|
|
||||||
|
<span class="sd"> * https://docs.python.org/3/library/sqlite3.html#sqlite3.connect</span>
|
||||||
|
<span class="sd"> * https://www.sqlite.org/uri.html</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">uri</span> <span class="o">=</span> <span class="s1">'file:'</span> <span class="o">+</span> <span class="n">database</span> <span class="o">+</span> <span class="s1">'?mode=ro'</span>
|
||||||
|
<span class="k">with</span> <span class="n">contextlib</span><span class="o">.</span><span class="n">closing</span><span class="p">(</span><span class="n">sqlite3</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">uri</span><span class="p">,</span> <span class="n">uri</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span> <span class="k">as</span> <span class="n">connect</span><span class="p">:</span>
|
||||||
|
<span class="n">connect</span><span class="o">.</span><span class="n">row_factory</span> <span class="o">=</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">Row</span>
|
||||||
|
<span class="k">with</span> <span class="n">contextlib</span><span class="o">.</span><span class="n">closing</span><span class="p">(</span><span class="n">connect</span><span class="o">.</span><span class="n">cursor</span><span class="p">())</span> <span class="k">as</span> <span class="n">cursor</span><span class="p">:</span>
|
||||||
|
<span class="k">yield</span> <span class="n">cursor</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">search</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">query_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'wildcard'</span><span class="p">:</span> <span class="sa">r</span><span class="s1">'%'</span> <span class="o">+</span> <span class="n">query</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">' '</span><span class="p">,</span> <span class="sa">r</span><span class="s1">'%'</span><span class="p">)</span> <span class="o">+</span> <span class="sa">r</span><span class="s1">'%'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'limit'</span><span class="p">:</span> <span class="n">limit</span><span class="p">,</span>
|
||||||
|
<span class="s1">'offset'</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">limit</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">query_to_run</span> <span class="o">=</span> <span class="n">query_str</span> <span class="o">+</span> <span class="s1">' LIMIT :limit OFFSET :offset'</span>
|
||||||
|
|
||||||
|
<span class="k">with</span> <span class="n">sqlite_cursor</span><span class="p">()</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">query_to_run</span><span class="p">,</span> <span class="n">query_params</span><span class="p">)</span>
|
||||||
|
<span class="n">col_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">cn</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">cn</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">description</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchall</span><span class="p">():</span>
|
||||||
|
<span class="n">kvmap</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">col_names</span><span class="p">,</span> <span class="nb">map</span><span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">row</span><span class="p">)))</span>
|
||||||
|
<span class="k">if</span> <span class="n">result_type</span> <span class="o">==</span> <span class="s2">"MainResult"</span><span class="p">:</span>
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="n">MainResult</span><span class="p">(</span><span class="o">**</span><span class="n">kvmap</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="n">KeyValue</span><span class="p">(</span><span class="n">kvmap</span><span class="o">=</span><span class="n">kvmap</span><span class="p">)</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
649
_modules/searx/engines/startpage.html
Normal file
@@ -0,0 +1,649 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.startpage — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.startpage</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.startpage</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Startpage's language & region selectors are a mess ..</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _startpage regions:</span>
|
||||||
|
|
||||||
|
<span class="sd">Startpage regions</span>
|
||||||
|
<span class="sd">=================</span>
|
||||||
|
|
||||||
|
<span class="sd">In the list of regions there are tags we need to map to common region tags::</span>
|
||||||
|
|
||||||
|
<span class="sd"> pt-BR_BR --> pt_BR</span>
|
||||||
|
<span class="sd"> zh-CN_CN --> zh_Hans_CN</span>
|
||||||
|
<span class="sd"> zh-TW_TW --> zh_Hant_TW</span>
|
||||||
|
<span class="sd"> zh-TW_HK --> zh_Hant_HK</span>
|
||||||
|
<span class="sd"> en-GB_GB --> en_GB</span>
|
||||||
|
|
||||||
|
<span class="sd">and there is at least one tag with a three letter language tag (ISO 639-2)::</span>
|
||||||
|
|
||||||
|
<span class="sd"> fil_PH --> fil_PH</span>
|
||||||
|
|
||||||
|
<span class="sd">The locale code ``no_NO`` from Startpage does not exists and is mapped to</span>
|
||||||
|
<span class="sd">``nb-NO``::</span>
|
||||||
|
|
||||||
|
<span class="sd"> babel.core.UnknownLocaleError: unknown locale 'no_NO'</span>
|
||||||
|
|
||||||
|
<span class="sd">For reference see languages-subtag at iana; ``no`` is the macrolanguage [1]_ and</span>
|
||||||
|
<span class="sd">W3C recommends subtag over macrolanguage [2]_.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. [1] `iana: language-subtag-registry</span>
|
||||||
|
<span class="sd"> <https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry>`_ ::</span>
|
||||||
|
|
||||||
|
<span class="sd"> type: language</span>
|
||||||
|
<span class="sd"> Subtag: nb</span>
|
||||||
|
<span class="sd"> Description: Norwegian Bokmål</span>
|
||||||
|
<span class="sd"> Added: 2005-10-16</span>
|
||||||
|
<span class="sd"> Suppress-Script: Latn</span>
|
||||||
|
<span class="sd"> Macrolanguage: no</span>
|
||||||
|
|
||||||
|
<span class="sd">.. [2]</span>
|
||||||
|
<span class="sd"> Use macrolanguages with care. Some language subtags have a Scope field set to</span>
|
||||||
|
<span class="sd"> macrolanguage, i.e. this primary language subtag encompasses a number of more</span>
|
||||||
|
<span class="sd"> specific primary language subtags in the registry. ... As we recommended for</span>
|
||||||
|
<span class="sd"> the collection subtags mentioned above, in most cases you should try to use</span>
|
||||||
|
<span class="sd"> the more specific subtags ... `W3: The primary language subtag</span>
|
||||||
|
<span class="sd"> <https://www.w3.org/International/questions/qa-choosing-language-tags#langsubtag>`_</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _startpage languages:</span>
|
||||||
|
|
||||||
|
<span class="sd">Startpage languages</span>
|
||||||
|
<span class="sd">===================</span>
|
||||||
|
|
||||||
|
<span class="sd">:py:obj:`send_accept_language_header`:</span>
|
||||||
|
<span class="sd"> The displayed name in Startpage's settings page depend on the location of the</span>
|
||||||
|
<span class="sd"> IP when ``Accept-Language`` HTTP header is unset. In :py:obj:`fetch_traits`</span>
|
||||||
|
<span class="sd"> we use::</span>
|
||||||
|
|
||||||
|
<span class="sd"> 'Accept-Language': "en-US,en;q=0.5",</span>
|
||||||
|
<span class="sd"> ..</span>
|
||||||
|
|
||||||
|
<span class="sd"> to get uniform names independent from the IP).</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _startpage categories:</span>
|
||||||
|
|
||||||
|
<span class="sd">Startpage categories</span>
|
||||||
|
<span class="sd">====================</span>
|
||||||
|
|
||||||
|
<span class="sd">Startpage's category (for Web-search, News, Videos, ..) is set by</span>
|
||||||
|
<span class="sd">:py:obj:`startpage_categ` in settings.yml::</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: startpage</span>
|
||||||
|
<span class="sd"> engine: startpage</span>
|
||||||
|
<span class="sd"> startpage_categ: web</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
|
||||||
|
<span class="sd">.. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> Supported categories are ``web``, ``news`` and ``images``.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-statements</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">unicodedata</span><span class="w"> </span><span class="kn">import</span> <span class="n">normalize</span><span class="p">,</span> <span class="n">combining</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">dateutil.parser</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">lxml.html</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.localedata</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extr</span><span class="p">,</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">gen_useragent</span><span class="p">,</span> <span class="n">html_to_text</span><span class="p">,</span> <span class="n">humanize_bytes</span><span class="p">,</span> <span class="n">remove_pua_from_str</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineCaptchaException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">region_tag</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineCache</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://startpage.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q2333295'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">startpage_categ</span> <span class="o">=</span> <span class="s1">'web'</span>
|
||||||
|
<span class="sd">"""Startpage's category, visit :ref:`startpage categories`.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="sd">"""Startpage tries to guess user's language and territory from the HTTP</span>
|
||||||
|
<span class="sd">``Accept-Language``. Optional the user can select a search-language (can be</span>
|
||||||
|
<span class="sd">different to the UI language) and a region filter.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">max_page</span> <span class="o">=</span> <span class="mi">18</span>
|
||||||
|
<span class="sd">"""Tested 18 pages maximum (argument ``page``), to be save max is set to 20."""</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'day'</span><span class="p">:</span> <span class="s1">'d'</span><span class="p">,</span> <span class="s1">'week'</span><span class="p">:</span> <span class="s1">'w'</span><span class="p">,</span> <span class="s1">'month'</span><span class="p">:</span> <span class="s1">'m'</span><span class="p">,</span> <span class="s1">'year'</span><span class="p">:</span> <span class="s1">'y'</span><span class="p">}</span>
|
||||||
|
<span class="n">safesearch_dict</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'0'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'0'</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># search-url</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://www.startpage.com'</span>
|
||||||
|
<span class="n">search_url</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">'/sp/search'</span>
|
||||||
|
|
||||||
|
<span class="c1"># specific xpath variables</span>
|
||||||
|
<span class="c1"># ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]</span>
|
||||||
|
<span class="c1"># not ads: div[@class="result"] are the direct children of div[@id="results"]</span>
|
||||||
|
<span class="n">search_form_xpath</span> <span class="o">=</span> <span class="s1">'//form[@id="search"]'</span>
|
||||||
|
<span class="sd">"""XPath of Startpage's origin search form</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code: html</span>
|
||||||
|
|
||||||
|
<span class="sd"> <form action="/sp/search" method="post"></span>
|
||||||
|
<span class="sd"> <input type="text" name="query" value="" ..></span>
|
||||||
|
<span class="sd"> <input type="hidden" name="t" value="device"></span>
|
||||||
|
<span class="sd"> <input type="hidden" name="lui" value="english"></span>
|
||||||
|
<span class="sd"> <input type="hidden" name="sc" value="Q7Mt5TRqowKB00"></span>
|
||||||
|
<span class="sd"> <input type="hidden" name="cat" value="web"></span>
|
||||||
|
<span class="sd"> <input type="hidden" class="abp" id="abp-input" name="abp" value="1"></span>
|
||||||
|
<span class="sd"> </form></span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">CACHE</span><span class="p">:</span> <span class="n">EngineCache</span>
|
||||||
|
<span class="sd">"""Persistent (SQLite) key/value cache that deletes its values after ``expire``</span>
|
||||||
|
<span class="sd">seconds."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">_</span><span class="p">):</span>
|
||||||
|
<span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
|
||||||
|
<span class="c1"># hint: all three startpage engines (WEB, Images & News) can/should use the</span>
|
||||||
|
<span class="c1"># same sc_code ..</span>
|
||||||
|
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">EngineCache</span><span class="p">(</span><span class="s2">"startpage"</span><span class="p">)</span> <span class="c1"># type:ignore</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">sc_code_cache_sec</span> <span class="o">=</span> <span class="mi">3600</span>
|
||||||
|
<span class="sd">"""Time in seconds the sc-code is cached in memory :py:obj:`get_sc_code`."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_sc_code">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/startpage.html#searx.engines.startpage.get_sc_code">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_sc_code</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get an actual ``sc`` argument from Startpage's search form (HTML page).</span>
|
||||||
|
|
||||||
|
<span class="sd"> Startpage puts a ``sc`` argument on every HTML :py:obj:`search form</span>
|
||||||
|
<span class="sd"> <search_form_xpath>`. Without this argument Startpage considers the request</span>
|
||||||
|
<span class="sd"> is from a bot. We do not know what is encoded in the value of the ``sc``</span>
|
||||||
|
<span class="sd"> argument, but it seems to be a kind of a *timestamp*.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Startpage's search form generates a new sc-code on each request. This</span>
|
||||||
|
<span class="sd"> function scrapes a new sc-code from Startpage's home page every</span>
|
||||||
|
<span class="sd"> :py:obj:`sc_code_cache_sec` seconds."""</span>
|
||||||
|
|
||||||
|
<span class="n">sc_code</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"SC_CODE"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">sc_code</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"get_sc_code: using cached value: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">sc_code</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">sc_code</span>
|
||||||
|
|
||||||
|
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="o">**</span><span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]}</span>
|
||||||
|
|
||||||
|
<span class="c1"># add Accept-Language header</span>
|
||||||
|
<span class="k">if</span> <span class="n">searxng_locale</span> <span class="o">==</span> <span class="s1">'all'</span><span class="p">:</span>
|
||||||
|
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="s1">'en-US'</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">send_accept_language_header</span><span class="p">:</span>
|
||||||
|
<span class="n">ac_lang</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
|
||||||
|
<span class="n">ac_lang</span> <span class="o">=</span> <span class="s2">"</span><span class="si">%s</span><span class="s2">-</span><span class="si">%s</span><span class="s2">,</span><span class="si">%s</span><span class="s2">;q=0.9,*;q=0.5"</span> <span class="o">%</span> <span class="p">(</span>
|
||||||
|
<span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="p">,</span>
|
||||||
|
<span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">,</span>
|
||||||
|
<span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">headers</span><span class="p">[</span><span class="s1">'Accept-Language'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ac_lang</span>
|
||||||
|
|
||||||
|
<span class="n">get_sc_url</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">'/'</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"get_sc_code: querying new sc timestamp @ </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">get_sc_url</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"get_sc_code: request headers: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">headers</span><span class="p">)</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">get_sc_url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># ?? x = network.get('https://www.startpage.com/sp/cdn/images/filter-chevron.svg', headers=headers)</span>
|
||||||
|
<span class="c1"># ?? https://www.startpage.com/sp/cdn/images/filter-chevron.svg</span>
|
||||||
|
<span class="c1"># ?? ping-back URL: https://www.startpage.com/sp/pb?sc=TLsB0oITjZ8F21</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">str</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'https://www.startpage.com/sp/captcha'</span><span class="p">):</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">(</span>
|
||||||
|
<span class="n">message</span><span class="o">=</span><span class="s2">"get_sc_code: got redirected to https://www.startpage.com/sp/captcha"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sc_code</span> <span class="o">=</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">search_form_xpath</span> <span class="o">+</span> <span class="s1">'//input[@name="sc"]/@value'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="k">except</span> <span class="ne">IndexError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"suspend startpage API --> https://github.com/searxng/searxng/pull/695"</span><span class="p">)</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">(</span>
|
||||||
|
<span class="n">message</span><span class="o">=</span><span class="s2">"get_sc_code: [PR-695] querying new sc timestamp failed! (</span><span class="si">%s</span><span class="s2">)"</span> <span class="o">%</span> <span class="n">resp</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="p">)</span> <span class="kn">from</span><span class="w"> </span><span class="nn">exc</span>
|
||||||
|
|
||||||
|
<span class="n">sc_code</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">sc_code</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"get_sc_code: new value is: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">sc_code</span><span class="p">)</span>
|
||||||
|
<span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="s2">"SC_CODE"</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">sc_code</span><span class="p">,</span> <span class="n">expire</span><span class="o">=</span><span class="n">sc_code_cache_sec</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">sc_code</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/startpage.html#searx.engines.startpage.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Assemble a Startpage request.</span>
|
||||||
|
|
||||||
|
<span class="sd"> To avoid CAPTCHAs we need to send a well formed HTTP POST request with a</span>
|
||||||
|
<span class="sd"> cookie. We need to form a request that is identical to the request built by</span>
|
||||||
|
<span class="sd"> Startpage's search form:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - in the cookie the **region** is selected</span>
|
||||||
|
<span class="sd"> - in the HTTP POST data the **language** is selected</span>
|
||||||
|
|
||||||
|
<span class="sd"> Additionally the arguments form Startpage's search form needs to be set in</span>
|
||||||
|
<span class="sd"> HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">engine_region</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="s1">'en-US'</span><span class="p">)</span>
|
||||||
|
<span class="n">engine_language</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="s1">'en'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Origin'</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Referer'</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">'/'</span>
|
||||||
|
|
||||||
|
<span class="c1"># Build form data</span>
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">,</span>
|
||||||
|
<span class="s1">'cat'</span><span class="p">:</span> <span class="n">startpage_categ</span><span class="p">,</span>
|
||||||
|
<span class="s1">'t'</span><span class="p">:</span> <span class="s1">'device'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'sc'</span><span class="p">:</span> <span class="n">get_sc_code</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="n">params</span><span class="p">),</span> <span class="c1"># hint: this func needs HTTP headers</span>
|
||||||
|
<span class="s1">'with_date'</span><span class="p">:</span> <span class="n">time_range_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">],</span> <span class="s1">''</span><span class="p">),</span>
|
||||||
|
<span class="s1">'abp'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'abd'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'abe'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">engine_language</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_language</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'lui'</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_language</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'page'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span>
|
||||||
|
<span class="n">args</span><span class="p">[</span><span class="s1">'segment'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'startpage.udog'</span>
|
||||||
|
|
||||||
|
<span class="c1"># Build cookie</span>
|
||||||
|
<span class="n">lang_homepage</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||||
|
<span class="n">cookie</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">()</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'date_time'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'world'</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'disable_family_filter'</span><span class="p">]</span> <span class="o">=</span> <span class="n">safesearch_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'disable_open_in_new_window'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'0'</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'enable_post_method'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'1'</span> <span class="c1"># hint: POST</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'enable_proxy_safety_suggest'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'1'</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'enable_stay_control'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'1'</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'instant_answers'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'1'</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'lang_homepage'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'s/device/</span><span class="si">%s</span><span class="s1">/'</span> <span class="o">%</span> <span class="n">lang_homepage</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'num_of_results'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'10'</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'suggestions'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'1'</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'wt_unit'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'celsius'</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">engine_language</span><span class="p">:</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_language</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'language_ui'</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_language</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">engine_region</span><span class="p">:</span>
|
||||||
|
<span class="n">cookie</span><span class="p">[</span><span class="s1">'search_results_region'</span><span class="p">]</span> <span class="o">=</span> <span class="n">engine_region</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'preferences'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'N1N'</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="s2">"</span><span class="si">%s</span><span class="s2">EEE</span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">cookie</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'cookie preferences: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'preferences'</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"data: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">]</span> <span class="o">=</span> <span class="n">args</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'method'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'POST'</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_published_date</span><span class="p">(</span><span class="n">content</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]:</span>
|
||||||
|
<span class="n">published_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># check if search result starts with something like: "2 Sep 2014 ... "</span>
|
||||||
|
<span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]</span><span class="si">{2}</span><span class="s2"> [0-9]</span><span class="si">{4}</span><span class="s2"> \.\.\. "</span><span class="p">,</span> <span class="n">content</span><span class="p">):</span>
|
||||||
|
<span class="n">date_pos</span> <span class="o">=</span> <span class="n">content</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'...'</span><span class="p">)</span> <span class="o">+</span> <span class="mi">4</span>
|
||||||
|
<span class="n">date_string</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="mi">0</span> <span class="p">:</span> <span class="n">date_pos</span> <span class="o">-</span> <span class="mi">5</span><span class="p">]</span>
|
||||||
|
<span class="c1"># fix content string</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="n">date_pos</span><span class="p">:]</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">published_date</span> <span class="o">=</span> <span class="n">dateutil</span><span class="o">.</span><span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">date_string</span><span class="p">,</span> <span class="n">dayfirst</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
|
||||||
|
<span class="c1"># check if search result starts with something like: "5 days ago ... "</span>
|
||||||
|
<span class="k">elif</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^[0-9]+ days? ago \.\.\. "</span><span class="p">,</span> <span class="n">content</span><span class="p">):</span>
|
||||||
|
<span class="n">date_pos</span> <span class="o">=</span> <span class="n">content</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'...'</span><span class="p">)</span> <span class="o">+</span> <span class="mi">4</span>
|
||||||
|
<span class="n">date_string</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="mi">0</span> <span class="p">:</span> <span class="n">date_pos</span> <span class="o">-</span> <span class="mi">5</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># calculate datetime</span>
|
||||||
|
<span class="n">published_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\d+'</span><span class="p">,</span> <span class="n">date_string</span><span class="p">)</span><span class="o">.</span><span class="n">group</span><span class="p">()))</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="c1"># fix content string</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="p">[</span><span class="n">date_pos</span><span class="p">:]</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">content</span><span class="p">,</span> <span class="n">published_date</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_get_web_result</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'description'</span><span class="p">))</span>
|
||||||
|
<span class="n">content</span><span class="p">,</span> <span class="n">publishedDate</span> <span class="o">=</span> <span class="n">_parse_published_date</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'clickUrl'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]),</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">publishedDate</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_get_news_result</span><span class="p">(</span><span class="n">result</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">remove_pua_from_str</span><span class="p">(</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]))</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">remove_pua_from_str</span><span class="p">(</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'description'</span><span class="p">)))</span>
|
||||||
|
|
||||||
|
<span class="n">publishedDate</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'date'</span><span class="p">):</span>
|
||||||
|
<span class="n">publishedDate</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">'date'</span><span class="p">]</span> <span class="o">/</span> <span class="mi">1000</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnailUrl</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'thumbnailUrl'</span><span class="p">):</span>
|
||||||
|
<span class="n">thumbnailUrl</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">result</span><span class="p">[</span><span class="s1">'thumbnailUrl'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'clickUrl'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">publishedDate</span><span class="p">,</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="n">thumbnailUrl</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_get_image_result</span><span class="p">(</span><span class="n">result</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'altClickUrl'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">url</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnailUrl</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'thumbnailUrl'</span><span class="p">):</span>
|
||||||
|
<span class="n">thumbnailUrl</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">result</span><span class="p">[</span><span class="s1">'thumbnailUrl'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">resolution</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'width'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'height'</span><span class="p">):</span>
|
||||||
|
<span class="n">resolution</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">'width'</span><span class="p">]</span><span class="si">}</span><span class="s2">x</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">'height'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
<span class="n">filesize</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'filesize'</span><span class="p">):</span>
|
||||||
|
<span class="n">size_str</span> <span class="o">=</span> <span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="nb">str</span><span class="o">.</span><span class="n">isdigit</span><span class="p">,</span> <span class="n">result</span><span class="p">[</span><span class="s1">'filesize'</span><span class="p">]))</span>
|
||||||
|
<span class="n">filesize</span> <span class="o">=</span> <span class="n">humanize_bytes</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">size_str</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'images.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]),</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="s1">''</span><span class="p">,</span>
|
||||||
|
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'rawImageUrl'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'thumbnail_src'</span><span class="p">:</span> <span class="n">thumbnailUrl</span><span class="p">,</span>
|
||||||
|
<span class="s1">'resolution'</span><span class="p">:</span> <span class="n">resolution</span><span class="p">,</span>
|
||||||
|
<span class="s1">'img_format'</span><span class="p">:</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'format'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'filesize'</span><span class="p">:</span> <span class="n">filesize</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="n">categ</span> <span class="o">=</span> <span class="n">startpage_categ</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span>
|
||||||
|
<span class="n">results_raw</span> <span class="o">=</span> <span class="s1">'{'</span> <span class="o">+</span> <span class="n">extr</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"React.createElement(UIStartpage.AppSerp</span><span class="si">{</span><span class="n">categ</span><span class="si">}</span><span class="s2">, </span><span class="se">{{</span><span class="s2">"</span><span class="p">,</span> <span class="s1">'}})'</span><span class="p">)</span> <span class="o">+</span> <span class="s1">'}}'</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'Location'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"https://www.startpage.com/sp/captcha"</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineCaptchaException</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">results_json</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">results_raw</span><span class="p">)</span>
|
||||||
|
<span class="n">results_obj</span> <span class="o">=</span> <span class="n">results_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'render'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'presenter'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'regions'</span><span class="p">,</span> <span class="p">{})</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">results_categ</span> <span class="ow">in</span> <span class="n">results_obj</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mainline'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">results_categ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'results'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="k">if</span> <span class="n">results_categ</span><span class="p">[</span><span class="s1">'display_type'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'web-google'</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_get_web_result</span><span class="p">(</span><span class="n">item</span><span class="p">))</span>
|
||||||
|
<span class="k">elif</span> <span class="n">results_categ</span><span class="p">[</span><span class="s1">'display_type'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'news-bing'</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_get_news_result</span><span class="p">(</span><span class="n">item</span><span class="p">))</span>
|
||||||
|
<span class="k">elif</span> <span class="s1">'images'</span> <span class="ow">in</span> <span class="n">results_categ</span><span class="p">[</span><span class="s1">'display_type'</span><span class="p">]:</span>
|
||||||
|
<span class="n">item</span> <span class="o">=</span> <span class="n">_get_image_result</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">item</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/startpage.html#searx.engines.startpage.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch :ref:`languages <startpage languages>` and :ref:`regions <startpage</span>
|
||||||
|
<span class="sd"> regions>` from Startpage."""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches</span>
|
||||||
|
|
||||||
|
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'User-Agent'</span><span class="p">:</span> <span class="n">gen_useragent</span><span class="p">(),</span>
|
||||||
|
<span class="s1">'Accept-Language'</span><span class="p">:</span> <span class="s2">"en-US,en;q=0.5"</span><span class="p">,</span> <span class="c1"># bing needs to set the English language</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="s1">'https://www.startpage.com/do/settings'</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from Startpage is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">lxml</span><span class="o">.</span><span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="c1"># regions</span>
|
||||||
|
|
||||||
|
<span class="n">sp_region_names</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">option</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//form[@name="settings"]//select[@name="search_results_region"]/option'</span><span class="p">):</span>
|
||||||
|
<span class="n">sp_region_names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">option</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'value'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="n">sp_region_names</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_tag</span> <span class="o">==</span> <span class="s1">'all'</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">babel_region_tag</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'no_NO'</span><span class="p">:</span> <span class="s1">'nb_NO'</span><span class="p">}</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">)</span> <span class="c1"># norway</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'-'</span> <span class="ow">in</span> <span class="n">babel_region_tag</span><span class="p">:</span>
|
||||||
|
<span class="n">l</span><span class="p">,</span> <span class="n">r</span> <span class="o">=</span> <span class="n">babel_region_tag</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="n">r</span> <span class="o">=</span> <span class="n">r</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">l</span> <span class="o">+</span> <span class="s1">'_'</span> <span class="o">+</span> <span class="n">r</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'_'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">babel_region_tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'_'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: can't determine babel locale of startpage's locale </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">eng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
|
||||||
|
<span class="c1"># languages</span>
|
||||||
|
|
||||||
|
<span class="n">catalog_engine2code</span> <span class="o">=</span> <span class="p">{</span><span class="n">name</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span> <span class="n">lang_code</span> <span class="k">for</span> <span class="n">lang_code</span><span class="p">,</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">(</span><span class="s1">'en'</span><span class="p">)</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
|
||||||
|
|
||||||
|
<span class="c1"># get the native name of every language known by babel</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">lang_code</span> <span class="ow">in</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">lang_code</span><span class="p">:</span> <span class="n">lang_code</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">babel</span><span class="o">.</span><span class="n">localedata</span><span class="o">.</span><span class="n">locale_identifiers</span><span class="p">()):</span>
|
||||||
|
<span class="n">native_name</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">(</span><span class="n">lang_code</span><span class="p">)</span><span class="o">.</span><span class="n">get_language_name</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">native_name</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"ERROR: language name of startpage's language </span><span class="si">{</span><span class="n">lang_code</span><span class="si">}</span><span class="s2"> is unknown by babel"</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">native_name</span> <span class="o">=</span> <span class="n">native_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||||
|
<span class="c1"># add native name exactly as it is</span>
|
||||||
|
<span class="n">catalog_engine2code</span><span class="p">[</span><span class="n">native_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang_code</span>
|
||||||
|
|
||||||
|
<span class="c1"># add "normalized" language name (i.e. français becomes francais and español becomes espanol)</span>
|
||||||
|
<span class="n">unaccented_name</span> <span class="o">=</span> <span class="s1">''</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">c</span><span class="p">:</span> <span class="ow">not</span> <span class="n">combining</span><span class="p">(</span><span class="n">c</span><span class="p">),</span> <span class="n">normalize</span><span class="p">(</span><span class="s1">'NFKD'</span><span class="p">,</span> <span class="n">native_name</span><span class="p">)))</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">unaccented_name</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">unaccented_name</span><span class="o">.</span><span class="n">encode</span><span class="p">()):</span>
|
||||||
|
<span class="c1"># add only if result is ascii (otherwise "normalization" didn't work)</span>
|
||||||
|
<span class="n">catalog_engine2code</span><span class="p">[</span><span class="n">unaccented_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">lang_code</span>
|
||||||
|
|
||||||
|
<span class="c1"># values that can't be determined by babel's languages names</span>
|
||||||
|
|
||||||
|
<span class="n">catalog_engine2code</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="c1"># traditional chinese used in ..</span>
|
||||||
|
<span class="s1">'fantizhengwen'</span><span class="p">:</span> <span class="s1">'zh_Hant'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># Korean alphabet</span>
|
||||||
|
<span class="s1">'hangul'</span><span class="p">:</span> <span class="s1">'ko'</span><span class="p">,</span>
|
||||||
|
<span class="c1"># Malayalam is one of 22 scheduled languages of India.</span>
|
||||||
|
<span class="s1">'malayam'</span><span class="p">:</span> <span class="s1">'ml'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'norsk'</span><span class="p">:</span> <span class="s1">'nb'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'sinhalese'</span><span class="p">:</span> <span class="s1">'si'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">skip_eng_tags</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'english_uk'</span><span class="p">,</span> <span class="c1"># SearXNG lang 'en' already maps to 'english'</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">option</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//form[@name="settings"]//select[@name="language"]/option'</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">option</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'value'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_tag</span> <span class="ow">in</span> <span class="n">skip_eng_tags</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">option</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">catalog_engine2code</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">sxng_tag</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">catalog_engine2code</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
335
_modules/searx/engines/tineye.html
Normal file
@@ -0,0 +1,335 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.tineye — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.tineye</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.tineye</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This engine implements *Tineye - reverse image search*</span>
|
||||||
|
|
||||||
|
<span class="sd">Using TinEye, you can search by image or perform what we call a reverse image</span>
|
||||||
|
<span class="sd">search. You can do that by uploading an image or searching by URL. You can also</span>
|
||||||
|
<span class="sd">simply drag and drop your images to start your search. TinEye constantly crawls</span>
|
||||||
|
<span class="sd">the web and adds images to its index. Today, the TinEye index is over 50.2</span>
|
||||||
|
<span class="sd">billion images `[tineye.com] <https://tineye.com/how>`_.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> This SearXNG engine only supports *'searching by URL'* and it does not use</span>
|
||||||
|
<span class="sd"> the official API `[api.tineye.com] <https://api.tineye.com/python/docs/>`_.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://tineye.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q2382535'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://api.tineye.com/python/docs/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">engine_type</span> <span class="o">=</span> <span class="s1">'online_url_search'</span>
|
||||||
|
<span class="sd">""":py:obj:`searx.search.processors.online_url_search`"""</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">safesearch</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s1">'https://tineye.com'</span>
|
||||||
|
<span class="n">search_string</span> <span class="o">=</span> <span class="s1">'/api/v1/result_json/?page=</span><span class="si">{page}</span><span class="s1">&</span><span class="si">{query}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="n">FORMAT_NOT_SUPPORTED</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span>
|
||||||
|
<span class="s2">"Could not read that image url. This may be due to an unsupported file"</span>
|
||||||
|
<span class="s2">" format. TinEye only supports images that are JPEG, PNG, GIF, BMP, TIFF or WebP."</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="sd">"""TinEye error message"""</span>
|
||||||
|
|
||||||
|
<span class="n">NO_SIGNATURE_ERROR</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span>
|
||||||
|
<span class="s2">"The image is too simple to find matches. TinEye requires a basic level of"</span>
|
||||||
|
<span class="s2">" visual detail to successfully identify matches."</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="sd">"""TinEye error message"""</span>
|
||||||
|
|
||||||
|
<span class="n">DOWNLOAD_ERROR</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"The image could not be downloaded."</span><span class="p">)</span>
|
||||||
|
<span class="sd">"""TinEye error message"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online_url_search/tineye.html#searx.engines.tineye.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Build TinEye HTTP request using ``search_urls`` of a :py:obj:`engine_type`."""</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'search_urls'</span><span class="p">][</span><span class="s1">'data:image'</span><span class="p">]:</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'search_urls'</span><span class="p">][</span><span class="s1">'data:image'</span><span class="p">]</span>
|
||||||
|
<span class="k">elif</span> <span class="n">params</span><span class="p">[</span><span class="s1">'search_urls'</span><span class="p">][</span><span class="s1">'http'</span><span class="p">]:</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'search_urls'</span><span class="p">][</span><span class="s1">'http'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"query URL: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">query</span><span class="p">)</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">query</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="c1"># see https://github.com/TinEye/pytineye/blob/main/pytineye/api.py</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">search_string</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">query</span><span class="o">=</span><span class="n">query</span><span class="p">,</span> <span class="n">page</span><span class="o">=</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'Connection'</span><span class="p">:</span> <span class="s1">'keep-alive'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'Host'</span><span class="p">:</span> <span class="s1">'tineye.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'DNT'</span><span class="p">:</span> <span class="s1">'1'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'TE'</span><span class="p">:</span> <span class="s1">'trailers'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="parse_tineye_match">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online_url_search/tineye.html#searx.engines.tineye.parse_tineye_match">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_tineye_match</span><span class="p">(</span><span class="n">match_json</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Takes parsed JSON from the API server and turns it into a :py:obj:`dict`</span>
|
||||||
|
<span class="sd"> object.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Attributes `(class Match) <https://github.com/TinEye/pytineye/blob/main/pytineye/api.py>`__</span>
|
||||||
|
|
||||||
|
<span class="sd"> - `image_url`, link to the result image.</span>
|
||||||
|
<span class="sd"> - `domain`, domain this result was found on.</span>
|
||||||
|
<span class="sd"> - `score`, a number (0 to 100) that indicates how closely the images match.</span>
|
||||||
|
<span class="sd"> - `width`, image width in pixels.</span>
|
||||||
|
<span class="sd"> - `height`, image height in pixels.</span>
|
||||||
|
<span class="sd"> - `size`, image area in pixels.</span>
|
||||||
|
<span class="sd"> - `format`, image format.</span>
|
||||||
|
<span class="sd"> - `filesize`, image size in bytes.</span>
|
||||||
|
<span class="sd"> - `overlay`, overlay URL.</span>
|
||||||
|
<span class="sd"> - `tags`, whether this match belongs to a collection or stock domain.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - `backlinks`, a list of Backlink objects pointing to the original websites</span>
|
||||||
|
<span class="sd"> and image URLs. List items are instances of :py:obj:`dict`, (`Backlink</span>
|
||||||
|
<span class="sd"> <https://github.com/TinEye/pytineye/blob/main/pytineye/api.py>`__):</span>
|
||||||
|
|
||||||
|
<span class="sd"> - `url`, the image URL to the image.</span>
|
||||||
|
<span class="sd"> - `backlink`, the original website URL.</span>
|
||||||
|
<span class="sd"> - `crawl_date`, the date the image was crawled.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="c1"># HINT: there exists an alternative backlink dict in the domains list / e.g.::</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># match_json['domains'][0]['backlinks']</span>
|
||||||
|
|
||||||
|
<span class="n">backlinks</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">if</span> <span class="s2">"backlinks"</span> <span class="ow">in</span> <span class="n">match_json</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">backlink_json</span> <span class="ow">in</span> <span class="n">match_json</span><span class="p">[</span><span class="s2">"backlinks"</span><span class="p">]:</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">backlink_json</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"crawl_date"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">crawl_date</span><span class="p">:</span>
|
||||||
|
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">crawl_date</span><span class="p">,</span> <span class="s1">'%Y-%m-</span><span class="si">%d</span><span class="s1">'</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">crawl_date</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">min</span>
|
||||||
|
|
||||||
|
<span class="n">backlinks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"url"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'backlink'</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"backlink"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'crawl_date'</span><span class="p">:</span> <span class="n">crawl_date</span><span class="p">,</span>
|
||||||
|
<span class="s1">'image_name'</span><span class="p">:</span> <span class="n">backlink_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"image_name"</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'image_url'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"image_url"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'domain'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"domain"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'score'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"score"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'width'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"width"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'height'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"height"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'size'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"size"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'image_format'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"format"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'filesize'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"filesize"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'overlay'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"overlay"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'tags'</span><span class="p">:</span> <span class="n">match_json</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"tags"</span><span class="p">),</span>
|
||||||
|
<span class="s1">'backlinks'</span><span class="p">:</span> <span class="n">backlinks</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online_url_search/tineye.html#searx.engines.tineye.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse HTTP response from TinEye."""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># handle the 422 client side errors, and the possible 400 status code error</span>
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="p">(</span><span class="mi">400</span><span class="p">,</span> <span class="mi">422</span><span class="p">):</span>
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="n">suggestions</span> <span class="o">=</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'suggestions'</span><span class="p">,</span> <span class="p">{})</span>
|
||||||
|
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'HTTP Status Code: </span><span class="si">{</span><span class="n">resp</span><span class="o">.</span><span class="n">status_code</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">422</span><span class="p">:</span>
|
||||||
|
<span class="n">s_key</span> <span class="o">=</span> <span class="n">suggestions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'key'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s2">"Invalid image URL"</span><span class="p">:</span>
|
||||||
|
<span class="c1"># test https://docs.searxng.org/_static/searxng-wordmark.svg</span>
|
||||||
|
<span class="n">message</span> <span class="o">=</span> <span class="n">FORMAT_NOT_SUPPORTED</span>
|
||||||
|
<span class="k">elif</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s1">'NO_SIGNATURE_ERROR'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># test https://pngimg.com/uploads/dot/dot_PNG4.png</span>
|
||||||
|
<span class="n">message</span> <span class="o">=</span> <span class="n">NO_SIGNATURE_ERROR</span>
|
||||||
|
<span class="k">elif</span> <span class="n">s_key</span> <span class="o">==</span> <span class="s1">'Download Error'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># test https://notexists</span>
|
||||||
|
<span class="n">message</span> <span class="o">=</span> <span class="n">DOWNLOAD_ERROR</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unknown suggestion key encountered: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">s_key</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span> <span class="c1"># 400</span>
|
||||||
|
<span class="n">description</span> <span class="o">=</span> <span class="n">suggestions</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'description'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">description</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||||
|
<span class="n">message</span> <span class="o">=</span> <span class="s1">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">description</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># see https://github.com/searxng/searxng/pull/1456#issuecomment-1193105023</span>
|
||||||
|
<span class="c1"># results.add(results.types.Answer(answer=message))</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="c1"># Raise for all other responses</span>
|
||||||
|
<span class="n">resp</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">match_json</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">'matches'</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="n">tineye_match</span> <span class="o">=</span> <span class="n">parse_tineye_match</span><span class="p">(</span><span class="n">match_json</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'backlinks'</span><span class="p">]:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">backlink</span> <span class="o">=</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'backlinks'</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'images.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'backlink'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'thumbnail_src'</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'image_url'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'source'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'image_name'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'format'</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'image_format'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'width'</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'width'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'height'</span><span class="p">:</span> <span class="n">tineye_match</span><span class="p">[</span><span class="s1">'height'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">backlink</span><span class="p">[</span><span class="s1">'crawl_date'</span><span class="p">],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># append number of results</span>
|
||||||
|
|
||||||
|
<span class="n">number_of_results</span> <span class="o">=</span> <span class="n">json_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'num_matches'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">number_of_results</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'number_of_results'</span><span class="p">:</span> <span class="n">number_of_results</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
363
_modules/searx/engines/torznab.html
Normal file
@@ -0,0 +1,363 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.torznab — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.torznab</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.torznab</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Torznab_ is an API specification that provides a standardized way to query</span>
|
||||||
|
<span class="sd">torrent site for content. It is used by a number of torrent applications,</span>
|
||||||
|
<span class="sd">including Prowlarr_ and Jackett_.</span>
|
||||||
|
|
||||||
|
<span class="sd">Using this engine together with Prowlarr_ or Jackett_ allows you to search</span>
|
||||||
|
<span class="sd">a huge number of torrent sites which are not directly supported.</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">``base_url``:</span>
|
||||||
|
<span class="sd"> Torznab endpoint URL.</span>
|
||||||
|
|
||||||
|
<span class="sd">``api_key``:</span>
|
||||||
|
<span class="sd"> The API key to use for authentication.</span>
|
||||||
|
|
||||||
|
<span class="sd">``torznab_categories``:</span>
|
||||||
|
<span class="sd"> The categories to use for searching. This is a list of category IDs. See</span>
|
||||||
|
<span class="sd"> Prowlarr-categories_ or Jackett-categories_ for more information.</span>
|
||||||
|
|
||||||
|
<span class="sd">``show_torrent_files``:</span>
|
||||||
|
<span class="sd"> Whether to show the torrent file in the search results. Be careful as using</span>
|
||||||
|
<span class="sd"> this with Prowlarr_ or Jackett_ leaks the API key. This should be used only</span>
|
||||||
|
<span class="sd"> if you are querying a Torznab endpoint without authentication or if the</span>
|
||||||
|
<span class="sd"> instance is private. Be aware that private trackers may ban you if you share</span>
|
||||||
|
<span class="sd"> the torrent file. Defaults to ``false``.</span>
|
||||||
|
|
||||||
|
<span class="sd">``show_magnet_links``:</span>
|
||||||
|
<span class="sd"> Whether to show the magnet link in the search results. Be aware that private</span>
|
||||||
|
<span class="sd"> trackers may ban you if you share the magnet link. Defaults to ``true``.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Torznab:</span>
|
||||||
|
<span class="sd"> https://torznab.github.io/spec-1.3-draft/index.html</span>
|
||||||
|
<span class="sd">.. _Prowlarr:</span>
|
||||||
|
<span class="sd"> https://github.com/Prowlarr/Prowlarr</span>
|
||||||
|
<span class="sd">.. _Jackett:</span>
|
||||||
|
<span class="sd"> https://github.com/Jackett/Jackett</span>
|
||||||
|
<span class="sd">.. _Prowlarr-categories:</span>
|
||||||
|
<span class="sd"> https://wiki.servarr.com/en/prowlarr/cardigann-yml-definition#categories</span>
|
||||||
|
<span class="sd">.. _Jackett-categories:</span>
|
||||||
|
<span class="sd"> https://github.com/Jackett/Jackett/wiki/Jackett-Categories</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">etree</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineAPIException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">humanize_bytes</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># engine settings</span>
|
||||||
|
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s2">"https://torznab.github.io/spec-1.3-draft"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'XML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">categories</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'files'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">time_range_support</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="c1"># defined in settings.yml</span>
|
||||||
|
<span class="c1"># example (Jackett): "http://localhost:9117/api/v2.0/indexers/all/results/torznab"</span>
|
||||||
|
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="n">api_key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="c1"># https://newznab.readthedocs.io/en/latest/misc/api/#predefined-categories</span>
|
||||||
|
<span class="n">torznab_categories</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">show_torrent_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">show_magnet_links</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="init">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialize the engine."""</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">base_url</span><span class="p">)</span> <span class="o"><</span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'missing torznab base_url'</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Build the request params."""</span>
|
||||||
|
<span class="n">search_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">base_url</span> <span class="o">+</span> <span class="s1">'?t=search&q=</span><span class="si">{search_query}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">api_key</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">search_url</span> <span class="o">+=</span> <span class="s1">'&apikey=</span><span class="si">{api_key}</span><span class="s1">'</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">torznab_categories</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">search_url</span> <span class="o">+=</span> <span class="s1">'&cat=</span><span class="si">{torznab_categories}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||||
|
<span class="n">search_query</span><span class="o">=</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">),</span> <span class="n">api_key</span><span class="o">=</span><span class="n">api_key</span><span class="p">,</span> <span class="n">torznab_categories</span><span class="o">=</span><span class="s2">","</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">torznab_categories</span><span class="p">])</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse the XML response and return a list of results."""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">search_results</span> <span class="o">=</span> <span class="n">etree</span><span class="o">.</span><span class="n">XML</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># handle errors: https://newznab.readthedocs.io/en/latest/misc/api/#newznab-error-codes</span>
|
||||||
|
<span class="k">if</span> <span class="n">search_results</span><span class="o">.</span><span class="n">tag</span> <span class="o">==</span> <span class="s2">"error"</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxEngineAPIException</span><span class="p">(</span><span class="n">search_results</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"description"</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">channel</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span> <span class="o">=</span> <span class="n">search_results</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">item</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">channel</span><span class="o">.</span><span class="n">iterfind</span><span class="p">(</span><span class="s1">'item'</span><span class="p">):</span>
|
||||||
|
<span class="n">result</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="n">build_result</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="build_result">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.build_result">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">build_result</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Build a result from a XML item."""</span>
|
||||||
|
|
||||||
|
<span class="c1"># extract attributes from XML</span>
|
||||||
|
<span class="c1"># see https://torznab.github.io/spec-1.3-draft/torznab/Specification-v1.3.html#predefined-attributes</span>
|
||||||
|
<span class="n">enclosure</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'enclosure'</span><span class="p">)</span>
|
||||||
|
<span class="n">enclosure_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">enclosure</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">enclosure_url</span> <span class="o">=</span> <span class="n">enclosure</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'url'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">filesize</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'size'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">filesize</span> <span class="ow">and</span> <span class="n">enclosure</span><span class="p">:</span>
|
||||||
|
<span class="n">filesize</span> <span class="o">=</span> <span class="n">enclosure</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'length'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">guid</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'guid'</span><span class="p">)</span>
|
||||||
|
<span class="n">comments</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'comments'</span><span class="p">)</span>
|
||||||
|
<span class="n">pubDate</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'pubDate'</span><span class="p">)</span>
|
||||||
|
<span class="n">seeders</span> <span class="o">=</span> <span class="n">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'seeders'</span><span class="p">)</span>
|
||||||
|
<span class="n">leechers</span> <span class="o">=</span> <span class="n">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'leechers'</span><span class="p">)</span>
|
||||||
|
<span class="n">peers</span> <span class="o">=</span> <span class="n">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'peers'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># map attributes to SearXNG result</span>
|
||||||
|
<span class="n">result</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'torrent.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'filesize'</span><span class="p">:</span> <span class="n">humanize_bytes</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">filesize</span><span class="p">))</span> <span class="k">if</span> <span class="n">filesize</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s1">'files'</span><span class="p">:</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'files'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'seed'</span><span class="p">:</span> <span class="n">seeders</span><span class="p">,</span>
|
||||||
|
<span class="s1">'leech'</span><span class="p">:</span> <span class="n">_map_leechers</span><span class="p">(</span><span class="n">leechers</span><span class="p">,</span> <span class="n">seeders</span><span class="p">,</span> <span class="n">peers</span><span class="p">),</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">_map_result_url</span><span class="p">(</span><span class="n">guid</span><span class="p">,</span> <span class="n">comments</span><span class="p">),</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">_map_published_date</span><span class="p">(</span><span class="n">pubDate</span><span class="p">),</span>
|
||||||
|
<span class="s1">'torrentfile'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s1">'magnetlink'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">link</span> <span class="o">=</span> <span class="n">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'link'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">show_torrent_files</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span><span class="p">[</span><span class="s1">'torrentfile'</span><span class="p">]</span> <span class="o">=</span> <span class="n">_map_torrent_file</span><span class="p">(</span><span class="n">link</span><span class="p">,</span> <span class="n">enclosure_url</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">show_magnet_links</span><span class="p">:</span>
|
||||||
|
<span class="n">magneturl</span> <span class="o">=</span> <span class="n">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'magneturl'</span><span class="p">)</span>
|
||||||
|
<span class="n">result</span><span class="p">[</span><span class="s1">'magnetlink'</span><span class="p">]</span> <span class="o">=</span> <span class="n">_map_magnet_link</span><span class="p">(</span><span class="n">magneturl</span><span class="p">,</span> <span class="n">guid</span><span class="p">,</span> <span class="n">enclosure_url</span><span class="p">,</span> <span class="n">link</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">result</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_map_result_url</span><span class="p">(</span><span class="n">guid</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">comments</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">guid</span> <span class="ow">and</span> <span class="n">guid</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'http'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">guid</span>
|
||||||
|
<span class="k">if</span> <span class="n">comments</span> <span class="ow">and</span> <span class="n">comments</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'http'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">comments</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_map_leechers</span><span class="p">(</span><span class="n">leechers</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">seeders</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">peers</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">leechers</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">leechers</span>
|
||||||
|
<span class="k">if</span> <span class="n">seeders</span> <span class="ow">and</span> <span class="n">peers</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">peers</span><span class="p">)</span> <span class="o">-</span> <span class="nb">int</span><span class="p">(</span><span class="n">seeders</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_map_published_date</span><span class="p">(</span><span class="n">pubDate</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">pubDate</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">pubDate</span><span class="p">,</span> <span class="s1">'</span><span class="si">%a</span><span class="s1">, </span><span class="si">%d</span><span class="s1"> %b %Y %H:%M:%S %z'</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"ignore exception (publishedDate): </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_map_torrent_file</span><span class="p">(</span><span class="n">link</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">enclosure_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">link</span> <span class="ow">and</span> <span class="n">link</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'http'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">link</span>
|
||||||
|
<span class="k">if</span> <span class="n">enclosure_url</span> <span class="ow">and</span> <span class="n">enclosure_url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'http'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">enclosure_url</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_map_magnet_link</span><span class="p">(</span>
|
||||||
|
<span class="n">magneturl</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="n">guid</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="n">enclosure_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="n">link</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">magneturl</span> <span class="ow">and</span> <span class="n">magneturl</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'magnet'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">magneturl</span>
|
||||||
|
<span class="k">if</span> <span class="n">guid</span> <span class="ow">and</span> <span class="n">guid</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'magnet'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">guid</span>
|
||||||
|
<span class="k">if</span> <span class="n">enclosure_url</span> <span class="ow">and</span> <span class="n">enclosure_url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'magnet'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">enclosure_url</span>
|
||||||
|
<span class="k">if</span> <span class="n">link</span> <span class="ow">and</span> <span class="n">link</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'magnet'</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">link</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_attribute">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.get_attribute">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span><span class="p">,</span> <span class="n">property_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get attribute from item."""</span>
|
||||||
|
<span class="n">property_element</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">property_name</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">property_element</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">property_element</span><span class="o">.</span><span class="n">text</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_torznab_attribute">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/torznab.html#searx.engines.torznab.get_torznab_attribute">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_torznab_attribute</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span><span class="p">,</span> <span class="n">attribute_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get torznab special attribute from item."""</span>
|
||||||
|
<span class="n">element</span><span class="p">:</span> <span class="n">etree</span><span class="o">.</span><span class="n">Element</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">find</span><span class="p">(</span>
|
||||||
|
<span class="s1">'.//torznab:attr[@name="</span><span class="si">{attribute_name}</span><span class="s1">"]'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">attribute_name</span><span class="o">=</span><span class="n">attribute_name</span><span class="p">),</span>
|
||||||
|
<span class="p">{</span><span class="s1">'torznab'</span><span class="p">:</span> <span class="s1">'http://torznab.com/schemas/2015/feed'</span><span class="p">},</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">element</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">element</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
297
_modules/searx/engines/tubearchivist.html
Normal file
@@ -0,0 +1,297 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.tubearchivist — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.tubearchivist</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.tubearchivist</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""`Tube Archivist`_ - *Your self hosted YouTube media server.*</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Tube Archivist: https://www.tubearchivist.com</span>
|
||||||
|
|
||||||
|
<span class="sd">This engine connects with a self-hosted instance of `Tube Archivist`_ to allow</span>
|
||||||
|
<span class="sd">searching for your hosted videos.</span>
|
||||||
|
|
||||||
|
<span class="sd">`Tube Archivist`_ (TA) requires authentication for all image loads via cookie</span>
|
||||||
|
<span class="sd">authentication. What this means is that by default, SearXNG will have no way to</span>
|
||||||
|
<span class="sd">pull images from TA (as there is no way to pass cookies in a URL string only).</span>
|
||||||
|
|
||||||
|
<span class="sd">In the meantime while work is done on the TA side, this can be worked around by</span>
|
||||||
|
<span class="sd">bypassing auth for images in TA by altering the default TA nginx file.</span>
|
||||||
|
|
||||||
|
<span class="sd">This is located in the main tubearchivist docker container at::</span>
|
||||||
|
|
||||||
|
<span class="sd"> /etc/nginx/sites-available/default</span>
|
||||||
|
|
||||||
|
<span class="sd">It is **strongly** recommended first setting up the intial connection and</span>
|
||||||
|
<span class="sd">verying searching works first with broken images, and then attempting this</span>
|
||||||
|
<span class="sd">change. This will limit any debugging to only images, rather than</span>
|
||||||
|
<span class="sd">tokens/networking.</span>
|
||||||
|
|
||||||
|
<span class="sd">Steps to enable **unauthenticated** metadata access for channels and videos:</span>
|
||||||
|
|
||||||
|
<span class="sd">#. Perform any backups of TA before editing core configurations.</span>
|
||||||
|
|
||||||
|
<span class="sd">#. Copy the contents of the file ``/etc/nginx/sites-available/default`` in the</span>
|
||||||
|
<span class="sd"> TA docker container</span>
|
||||||
|
|
||||||
|
<span class="sd">#. Edit ``location /cache/videos`` and ``location /cache/channels``. Comment</span>
|
||||||
|
<span class="sd"> out the line ``auth_request /api/ping/;`` to ``# auth_request /api/ping/;``.</span>
|
||||||
|
|
||||||
|
<span class="sd">#. Save the file to wherever you normally store your docker configuration.</span>
|
||||||
|
|
||||||
|
<span class="sd">#. Mount this new configuration over the default configuration. With ``docker</span>
|
||||||
|
<span class="sd"> run``, this would be::</span>
|
||||||
|
|
||||||
|
<span class="sd"> -v ./your-new-config.yml:/etc/nginx/sites-available/default</span>
|
||||||
|
|
||||||
|
<span class="sd"> With ``docker compose``, this would be::</span>
|
||||||
|
|
||||||
|
<span class="sd"> - "./your-new-config.yml:/etc/nginx/sites-available/default:ro"</span>
|
||||||
|
|
||||||
|
<span class="sd">#. Start the TA container.</span>
|
||||||
|
|
||||||
|
<span class="sd">After these steps, double check that TA works as normal (nothing should be</span>
|
||||||
|
<span class="sd">different on the TA side). Searching again should now show images.</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following required settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`base_url`</span>
|
||||||
|
<span class="sd">- :py:obj:`ta_token`</span>
|
||||||
|
|
||||||
|
<span class="sd">Optional settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`ta_link_to_mp4`</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: tubearchivist</span>
|
||||||
|
<span class="sd"> engine: tubearchivist</span>
|
||||||
|
<span class="sd"> shortcut: tuba</span>
|
||||||
|
<span class="sd"> base_url:</span>
|
||||||
|
<span class="sd"> ta_token:</span>
|
||||||
|
<span class="sd"> ta_link_to_mp4: true</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">html_to_text</span><span class="p">,</span> <span class="n">humanize_number</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="c1"># pylint: disable=line-too-long</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.tubearchivist.com'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://docs.tubearchivist.com/api/introduction/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"videos"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Base URL of the Tube Archivist instance. Fill this in with your own</span>
|
||||||
|
<span class="sd">Tube Archivist URL (``http://your-instance:port``)."""</span>
|
||||||
|
|
||||||
|
<span class="n">ta_token</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""The API key to use for Authorization_ header. Can be found under:</span>
|
||||||
|
|
||||||
|
<span class="sd"> :menuselection:`Settings --> User --> Admin Interface`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Authorization: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Authorization</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">ta_link_to_mp4</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">"""Optional, if true SearXNG will link directly to the mp4 of the video to play</span>
|
||||||
|
<span class="sd">in the browser. The default behavior is to link into TubeArchivist's interface</span>
|
||||||
|
<span class="sd">directly."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">absolute_url</span><span class="p">(</span><span class="n">relative_url</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span><span class="si">}{</span><span class="n">relative_url</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">_</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">base_url</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'tubearchivist engine: base_url is unset'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ta_token</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'tubearchivist engine: ta_token is unset'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">query</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">args</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span><span class="si">}</span><span class="s2">/api/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">args</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">][</span><span class="s1">'Authorization'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'Token </span><span class="si">{</span><span class="n">ta_token</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">,</span> <span class="n">results</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="video_response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/tubearchivist.html#searx.engines.tubearchivist.video_response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">video_response</span><span class="p">(</span><span class="n">resp</span><span class="p">,</span> <span class="n">results</span><span class="p">:</span> <span class="n">EngineResults</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parse video response from Tubearchivist instances."""</span>
|
||||||
|
|
||||||
|
<span class="n">json_data</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s1">'results'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">channel_result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">'results'</span><span class="p">][</span><span class="s1">'channel_results'</span><span class="p">]:</span>
|
||||||
|
<span class="n">channel_url</span> <span class="o">=</span> <span class="n">absolute_url</span><span class="p">(</span><span class="sa">f</span><span class="s1">'/channel/</span><span class="si">{</span><span class="n">channel_result</span><span class="p">[</span><span class="s2">"channel_id"</span><span class="p">]</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">MainResult</span><span class="p">(</span>
|
||||||
|
<span class="n">url</span><span class="o">=</span><span class="n">channel_url</span><span class="p">,</span>
|
||||||
|
<span class="n">title</span><span class="o">=</span><span class="n">channel_result</span><span class="p">[</span><span class="s1">'channel_name'</span><span class="p">],</span>
|
||||||
|
<span class="n">content</span><span class="o">=</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">channel_result</span><span class="p">[</span><span class="s1">'channel_description'</span><span class="p">]),</span>
|
||||||
|
<span class="n">author</span><span class="o">=</span><span class="n">channel_result</span><span class="p">[</span><span class="s1">'channel_name'</span><span class="p">],</span>
|
||||||
|
<span class="n">views</span><span class="o">=</span><span class="n">humanize_number</span><span class="p">(</span><span class="n">channel_result</span><span class="p">[</span><span class="s1">'channel_subs'</span><span class="p">]),</span>
|
||||||
|
<span class="n">thumbnail</span><span class="o">=</span><span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">absolute_url</span><span class="p">(</span><span class="n">channel_result</span><span class="p">[</span><span class="s2">"channel_thumb_url"</span><span class="p">])</span><span class="si">}</span><span class="s1">?auth=</span><span class="si">{</span><span class="n">ta_token</span><span class="si">}</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">result</span><span class="o">=</span><span class="n">res</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">video_result</span> <span class="ow">in</span> <span class="n">json_data</span><span class="p">[</span><span class="s1">'results'</span><span class="p">][</span><span class="s1">'video_results'</span><span class="p">]:</span>
|
||||||
|
<span class="n">metadata</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[</span><span class="n">video_result</span><span class="p">[</span><span class="s1">'channel'</span><span class="p">][</span><span class="s1">'channel_name'</span><span class="p">],</span> <span class="o">*</span><span class="n">video_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tags'</span><span class="p">,</span> <span class="p">[])]))[:</span><span class="mi">5</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">ta_link_to_mp4</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span><span class="si">}{</span><span class="n">video_result</span><span class="p">[</span><span class="s2">"media_url"</span><span class="p">]</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">base_url</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"/"</span><span class="p">)</span><span class="si">}</span><span class="s1">/?videoId=</span><span class="si">{</span><span class="n">video_result</span><span class="p">[</span><span class="s2">"youtube_id"</span><span class="p">]</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
|
||||||
|
<span class="c1"># a type for the video.html template is not yet implemented</span>
|
||||||
|
<span class="c1"># --> using LegacyResult</span>
|
||||||
|
|
||||||
|
<span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'videos.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">video_result</span><span class="p">[</span><span class="s1">'title'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">html_to_text</span><span class="p">(</span><span class="n">video_result</span><span class="p">[</span><span class="s1">'description'</span><span class="p">]),</span>
|
||||||
|
<span class="s1">'author'</span><span class="p">:</span> <span class="n">video_result</span><span class="p">[</span><span class="s1">'channel'</span><span class="p">][</span><span class="s1">'channel_name'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'length'</span><span class="p">:</span> <span class="n">video_result</span><span class="p">[</span><span class="s1">'player'</span><span class="p">][</span><span class="s1">'duration_str'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'views'</span><span class="p">:</span> <span class="n">humanize_number</span><span class="p">(</span><span class="n">video_result</span><span class="p">[</span><span class="s1">'stats'</span><span class="p">][</span><span class="s1">'view_count'</span><span class="p">]),</span>
|
||||||
|
<span class="s1">'publishedDate'</span><span class="p">:</span> <span class="n">parse</span><span class="p">(</span><span class="n">video_result</span><span class="p">[</span><span class="s1">'published'</span><span class="p">]),</span>
|
||||||
|
<span class="s1">'thumbnail'</span><span class="p">:</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">absolute_url</span><span class="p">(</span><span class="n">video_result</span><span class="p">[</span><span class="s2">"vid_thumb_url"</span><span class="p">])</span><span class="si">}</span><span class="s1">?auth=</span><span class="si">{</span><span class="n">ta_token</span><span class="si">}</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'metadata'</span><span class="p">:</span> <span class="s1">' | '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">metadata</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">LegacyResult</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">))</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
201
_modules/searx/engines/voidlinux.html
Normal file
@@ -0,0 +1,201 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.voidlinux — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.voidlinux</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.voidlinux</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""SearXNG engine for `Void Linux binary packages`_. Void is a general purpose</span>
|
||||||
|
<span class="sd">operating system, based on the monolithic Linux kernel. Its package system</span>
|
||||||
|
<span class="sd">allows you to quickly install, update and remove software; software is provided</span>
|
||||||
|
<span class="sd">in binary packages or can be built directly from sources with the help of the</span>
|
||||||
|
<span class="sd">XBPS source packages collection.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Void Linux binary packages: https://voidlinux.org/packages/</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote_plus</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">humanize_bytes</span>
|
||||||
|
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'website'</span><span class="p">:</span> <span class="s1">'https://voidlinux.org/packages/'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'wikidata_id'</span><span class="p">:</span> <span class="s1">'Q19310966'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'use_official_api'</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s1">'official_api_documentation'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s1">'require_api_key'</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s1">'results'</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'packages'</span><span class="p">,</span> <span class="s1">'it'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">base_url</span> <span class="o">=</span> <span class="s2">"https://xq-api.voidlinux.org"</span>
|
||||||
|
<span class="n">pkg_repo_url</span> <span class="o">=</span> <span class="s2">"https://github.com/void-linux/void-packages"</span>
|
||||||
|
|
||||||
|
<span class="n">void_arch</span> <span class="o">=</span> <span class="s1">'x86_64'</span>
|
||||||
|
<span class="sd">"""Default architecture to search for. For valid values see :py:obj:`ARCH_RE`"""</span>
|
||||||
|
|
||||||
|
<span class="n">ARCH_RE</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s1">'aarch64-musl|armv6l-musl|armv7l-musl|x86_64-musl|aarch64|armv6l|armv7l|i686|x86_64'</span><span class="p">)</span>
|
||||||
|
<span class="sd">"""Regular expression that match a architecture in the query string."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="n">arch_path</span> <span class="o">=</span> <span class="n">ARCH_RE</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">arch_path</span><span class="p">:</span>
|
||||||
|
<span class="n">arch_path</span> <span class="o">=</span> <span class="n">arch_path</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">arch_path</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">arch_path</span> <span class="o">=</span> <span class="n">void_arch</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">/v1/query/</span><span class="si">{</span><span class="n">arch_path</span><span class="si">}</span><span class="s2">?q=</span><span class="si">{</span><span class="n">quote_plus</span><span class="p">(</span><span class="n">query</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/void.html#searx.engines.voidlinux.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""</span>
|
||||||
|
<span class="sd"> At Void Linux, several packages sometimes share the same source code</span>
|
||||||
|
<span class="sd"> (template) and therefore also have the same URL. Results with identical</span>
|
||||||
|
<span class="sd"> URLs are merged as one result for SearXNG.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">packages</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()[</span><span class="s1">'data'</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="c1"># 32bit and dbg packages don't have their own package templates</span>
|
||||||
|
<span class="n">github_slug</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">"-(32bit|dbg)$"</span><span class="p">,</span> <span class="s2">""</span><span class="p">,</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">])</span>
|
||||||
|
<span class="n">pkg_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">pkg_repo_url</span><span class="si">}</span><span class="s2">/tree/master/srcpkgs/</span><span class="si">{</span><span class="n">github_slug</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
<span class="n">pkg_list</span> <span class="o">=</span> <span class="n">packages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">pkg_url</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="n">pkg_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">'short_desc'</span><span class="p">]</span><span class="si">}</span><span class="s2"> - </span><span class="si">{</span><span class="n">humanize_bytes</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="s1">'filename_size'</span><span class="p">])</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="s1">'package_name'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'version'</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"v</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">'version'</span><span class="p">]</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">result</span><span class="p">[</span><span class="s1">'revision'</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="s1">'tags'</span><span class="p">:</span> <span class="n">result</span><span class="p">[</span><span class="s1">'repository'</span><span class="p">],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">packages</span><span class="p">[</span><span class="n">pkg_url</span><span class="p">]</span> <span class="o">=</span> <span class="n">pkg_list</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">pkg_url</span><span class="p">,</span> <span class="n">pkg_list</span> <span class="ow">in</span> <span class="n">packages</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">pkg_url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'template'</span><span class="p">:</span> <span class="s1">'packages.html'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="s1">' | '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="s1">'title'</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">pkg_list</span><span class="p">),</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">pkg_list</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'content'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'package_name'</span><span class="p">:</span> <span class="s1">' | '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">x</span><span class="p">[</span><span class="s1">'package_name'</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">pkg_list</span><span class="p">),</span>
|
||||||
|
<span class="s1">'version'</span><span class="p">:</span> <span class="n">pkg_list</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'version'</span><span class="p">],</span>
|
||||||
|
<span class="s1">'tags'</span><span class="p">:</span> <span class="p">[</span><span class="n">x</span><span class="p">[</span><span class="s1">'tags'</span><span class="p">]</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">pkg_list</span><span class="p">],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
938
_modules/searx/engines/wikidata.html
Normal file
@@ -0,0 +1,938 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.wikidata — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.wikidata</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.wikidata</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This module implements the Wikidata engine. Some implementations are shared</span>
|
||||||
|
<span class="sd">from :ref:`wikipedia engine`.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-class-docstring</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">hashlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">md5</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span><span class="p">,</span> <span class="n">unquote</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">loads</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">isoparse</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">babel.dates</span><span class="w"> </span><span class="kn">import</span> <span class="n">format_datetime</span><span class="p">,</span> <span class="n">format_date</span><span class="p">,</span> <span class="n">format_time</span><span class="p">,</span> <span class="n">get_datetime_format</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">WIKIDATA_UNITS</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">post</span><span class="p">,</span> <span class="n">get</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">searxng_useragent</span><span class="p">,</span> <span class="n">get_string_replaces_function</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.external_urls</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_external_url</span><span class="p">,</span> <span class="n">get_earth_coordinates_url</span><span class="p">,</span> <span class="n">area_to_osm_zoom</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.engines.wikipedia</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">fetch_wikimedia_traits</span><span class="p">,</span>
|
||||||
|
<span class="n">get_wiki_params</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://wikidata.org/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q2013'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://query.wikidata.org/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">display_type</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"infobox"</span><span class="p">]</span>
|
||||||
|
<span class="sd">"""A list of display types composed from ``infobox`` and ``list``. The latter</span>
|
||||||
|
<span class="sd">one will add a hit to the result list. The first one will show a hit in the</span>
|
||||||
|
<span class="sd">info box. Both values can be set, or one of the two can be set."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># SPARQL</span>
|
||||||
|
<span class="n">SPARQL_ENDPOINT_URL</span> <span class="o">=</span> <span class="s1">'https://query.wikidata.org/sparql'</span>
|
||||||
|
<span class="n">SPARQL_EXPLAIN_URL</span> <span class="o">=</span> <span class="s1">'https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain'</span>
|
||||||
|
<span class="n">WIKIDATA_PROPERTIES</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'P434'</span><span class="p">:</span> <span class="s1">'MusicBrainz'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P435'</span><span class="p">:</span> <span class="s1">'MusicBrainz'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P436'</span><span class="p">:</span> <span class="s1">'MusicBrainz'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P966'</span><span class="p">:</span> <span class="s1">'MusicBrainz'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P345'</span><span class="p">:</span> <span class="s1">'IMDb'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P2397'</span><span class="p">:</span> <span class="s1">'YouTube'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P1651'</span><span class="p">:</span> <span class="s1">'YouTube'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P2002'</span><span class="p">:</span> <span class="s1">'Twitter'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P2013'</span><span class="p">:</span> <span class="s1">'Facebook'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P2003'</span><span class="p">:</span> <span class="s1">'Instagram'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P4033'</span><span class="p">:</span> <span class="s1">'Mastodon'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P11947'</span><span class="p">:</span> <span class="s1">'Lemmy'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'P12622'</span><span class="p">:</span> <span class="s1">'PeerTube'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># SERVICE wikibase:mwapi : https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual/MWAPI</span>
|
||||||
|
<span class="c1"># SERVICE wikibase:label: https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_Label#Manual_Label_SERVICE</span>
|
||||||
|
<span class="c1"># https://en.wikibooks.org/wiki/SPARQL/WIKIDATA_Precision,_Units_and_Coordinates</span>
|
||||||
|
<span class="c1"># https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format#Data_model</span>
|
||||||
|
<span class="c1"># optimization:</span>
|
||||||
|
<span class="c1"># * https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/query_optimization</span>
|
||||||
|
<span class="c1"># * https://github.com/blazegraph/database/wiki/QueryHints</span>
|
||||||
|
<span class="n">QUERY_TEMPLATE</span> <span class="o">=</span> <span class="s2">"""</span>
|
||||||
|
<span class="s2">SELECT ?item ?itemLabel ?itemDescription ?lat ?long %SELECT%</span>
|
||||||
|
<span class="s2">WHERE</span>
|
||||||
|
<span class="s2">{</span>
|
||||||
|
<span class="s2"> SERVICE wikibase:mwapi {</span>
|
||||||
|
<span class="s2"> bd:serviceParam wikibase:endpoint "www.wikidata.org";</span>
|
||||||
|
<span class="s2"> wikibase:api "EntitySearch";</span>
|
||||||
|
<span class="s2"> wikibase:limit 1;</span>
|
||||||
|
<span class="s2"> mwapi:search "%QUERY%";</span>
|
||||||
|
<span class="s2"> mwapi:language "%LANGUAGE%".</span>
|
||||||
|
<span class="s2"> ?item wikibase:apiOutputItem mwapi:item.</span>
|
||||||
|
<span class="s2"> }</span>
|
||||||
|
<span class="s2"> hint:Prior hint:runFirst "true".</span>
|
||||||
|
|
||||||
|
<span class="s2"> %WHERE%</span>
|
||||||
|
|
||||||
|
<span class="s2"> SERVICE wikibase:label {</span>
|
||||||
|
<span class="s2"> bd:serviceParam wikibase:language "%LANGUAGE%,en".</span>
|
||||||
|
<span class="s2"> ?item rdfs:label ?itemLabel .</span>
|
||||||
|
<span class="s2"> ?item schema:description ?itemDescription .</span>
|
||||||
|
<span class="s2"> %WIKIBASE_LABELS%</span>
|
||||||
|
<span class="s2"> }</span>
|
||||||
|
|
||||||
|
<span class="s2">}</span>
|
||||||
|
<span class="s2">GROUP BY ?item ?itemLabel ?itemDescription ?lat ?long </span><span class="si">%G</span><span class="s2">ROUP_BY%</span>
|
||||||
|
<span class="s2">"""</span>
|
||||||
|
|
||||||
|
<span class="c1"># Get the calendar names and the property names</span>
|
||||||
|
<span class="n">QUERY_PROPERTY_NAMES</span> <span class="o">=</span> <span class="s2">"""</span>
|
||||||
|
<span class="s2">SELECT ?item ?name</span>
|
||||||
|
<span class="s2">WHERE {</span>
|
||||||
|
<span class="s2"> {</span>
|
||||||
|
<span class="s2"> SELECT ?item</span>
|
||||||
|
<span class="s2"> WHERE { ?item wdt:P279* wd:Q12132 }</span>
|
||||||
|
<span class="s2"> } UNION {</span>
|
||||||
|
<span class="s2"> VALUES ?item { %ATTRIBUTES% }</span>
|
||||||
|
<span class="s2"> }</span>
|
||||||
|
<span class="s2"> OPTIONAL { ?item rdfs:label ?name. }</span>
|
||||||
|
<span class="s2">}</span>
|
||||||
|
<span class="s2">"""</span>
|
||||||
|
|
||||||
|
<span class="c1"># see the property "dummy value" of https://www.wikidata.org/wiki/Q2013 (Wikidata)</span>
|
||||||
|
<span class="c1"># hard coded here to avoid to an additional SPARQL request when the server starts</span>
|
||||||
|
<span class="n">DUMMY_ENTITY_URLS</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span>
|
||||||
|
<span class="s2">"http://www.wikidata.org/entity/"</span> <span class="o">+</span> <span class="n">wid</span> <span class="k">for</span> <span class="n">wid</span> <span class="ow">in</span> <span class="p">(</span><span class="s2">"Q4115189"</span><span class="p">,</span> <span class="s2">"Q13406268"</span><span class="p">,</span> <span class="s2">"Q15397819"</span><span class="p">,</span> <span class="s2">"Q17339402"</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># https://www.w3.org/TR/sparql11-query/#rSTRING_LITERAL1</span>
|
||||||
|
<span class="c1"># https://lists.w3.org/Archives/Public/public-rdf-dawg/2011OctDec/0175.html</span>
|
||||||
|
<span class="n">sparql_string_escape</span> <span class="o">=</span> <span class="n">get_string_replaces_function</span><span class="p">(</span>
|
||||||
|
<span class="c1"># fmt: off</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'</span><span class="se">\t</span><span class="s1">'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\\\t</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\\\n</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'</span><span class="se">\r</span><span class="s1">'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\\\r</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'</span><span class="se">\b</span><span class="s1">'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\\\b</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'</span><span class="se">\f</span><span class="s1">'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\\\f</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'</span><span class="se">\"</span><span class="s1">'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\\\"</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'</span><span class="se">\'</span><span class="s1">'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\\\'</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'</span><span class="se">\\</span><span class="s1">'</span><span class="p">:</span> <span class="s1">'</span><span class="se">\\\\</span><span class="s1">'</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="c1"># fmt: on</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">replace_http_by_https</span> <span class="o">=</span> <span class="n">get_string_replaces_function</span><span class="p">({</span><span class="s1">'http:'</span><span class="p">:</span> <span class="s1">'https:'</span><span class="p">})</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_headers</span><span class="p">():</span>
|
||||||
|
<span class="c1"># user agent: https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits</span>
|
||||||
|
<span class="k">return</span> <span class="p">{</span><span class="s1">'Accept'</span><span class="p">:</span> <span class="s1">'application/sparql-results+json'</span><span class="p">,</span> <span class="s1">'User-Agent'</span><span class="p">:</span> <span class="n">searxng_useragent</span><span class="p">()}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_label_for_entity</span><span class="p">(</span><span class="n">entity_id</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">entity_id</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="o">.</span><span class="n">get</span><span class="p">((</span><span class="n">entity_id</span><span class="p">,</span> <span class="n">language</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="o">.</span><span class="n">get</span><span class="p">((</span><span class="n">entity_id</span><span class="p">,</span> <span class="n">language</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]))</span>
|
||||||
|
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="o">.</span><span class="n">get</span><span class="p">((</span><span class="n">entity_id</span><span class="p">,</span> <span class="s1">'en'</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="n">entity_id</span>
|
||||||
|
<span class="k">return</span> <span class="n">name</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">send_wikidata_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s1">'GET'</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">'GET'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># query will be cached by wikidata</span>
|
||||||
|
<span class="n">http_response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">SPARQL_ENDPOINT_URL</span> <span class="o">+</span> <span class="s1">'?'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">}),</span> <span class="n">headers</span><span class="o">=</span><span class="n">get_headers</span><span class="p">(),</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="c1"># query won't be cached by wikidata</span>
|
||||||
|
<span class="n">http_response</span> <span class="o">=</span> <span class="n">post</span><span class="p">(</span><span class="n">SPARQL_ENDPOINT_URL</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="p">{</span><span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">},</span> <span class="n">headers</span><span class="o">=</span><span class="n">get_headers</span><span class="p">(),</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">http_response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'SPARQL endpoint error </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">http_response</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'request time </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">http_response</span><span class="o">.</span><span class="n">elapsed</span><span class="p">))</span>
|
||||||
|
<span class="n">http_response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||||
|
<span class="k">return</span> <span class="n">loads</span><span class="p">(</span><span class="n">http_response</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">eng_tag</span><span class="p">,</span> <span class="n">_wiki_netloc</span> <span class="o">=</span> <span class="n">get_wiki_params</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="n">traits</span><span class="p">)</span>
|
||||||
|
<span class="n">query</span><span class="p">,</span> <span class="n">attributes</span> <span class="o">=</span> <span class="n">get_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"request --> language </span><span class="si">%s</span><span class="s2"> // len(attributes): </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">attributes</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'method'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'POST'</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SPARQL_ENDPOINT_URL</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_headers</span><span class="p">()</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'attributes'</span><span class="p">]</span> <span class="o">=</span> <span class="n">attributes</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">jsonresponse</span> <span class="o">=</span> <span class="n">loads</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">decode</span><span class="p">())</span>
|
||||||
|
|
||||||
|
<span class="n">language</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span>
|
||||||
|
<span class="n">attributes</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">'attributes'</span><span class="p">]</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"request --> language </span><span class="si">%s</span><span class="s2"> // len(attributes): </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">language</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">attributes</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">seen_entities</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">jsonresponse</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'results'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'bindings'</span><span class="p">,</span> <span class="p">[]):</span>
|
||||||
|
<span class="n">attribute_result</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="n">value</span><span class="p">[</span><span class="s1">'value'</span><span class="p">]</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">result</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
|
||||||
|
<span class="n">entity_url</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="p">[</span><span class="s1">'item'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">entity_url</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">seen_entities</span> <span class="ow">and</span> <span class="n">entity_url</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">DUMMY_ENTITY_URLS</span><span class="p">:</span>
|
||||||
|
<span class="n">seen_entities</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">entity_url</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span> <span class="o">+=</span> <span class="n">get_results</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">,</span> <span class="n">attributes</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'The SPARQL request returns duplicate entities: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">_IMG_SRC_DEFAULT_URL_PREFIX</span> <span class="o">=</span> <span class="s2">"https://commons.wikimedia.org/wiki/Special:FilePath/"</span>
|
||||||
|
<span class="n">_IMG_SRC_NEW_URL_PREFIX</span> <span class="o">=</span> <span class="s2">"https://upload.wikimedia.org/wikipedia/commons/thumb/"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_thumbnail">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikidata.get_thumbnail">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_thumbnail</span><span class="p">(</span><span class="n">img_src</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Get Thumbnail image from wikimedia commons</span>
|
||||||
|
|
||||||
|
<span class="sd"> Images from commons.wikimedia.org are (HTTP) redirected to</span>
|
||||||
|
<span class="sd"> upload.wikimedia.org. The redirected URL can be calculated by this</span>
|
||||||
|
<span class="sd"> function.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - https://stackoverflow.com/a/33691240</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'get_thumbnail(): </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">img_src</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">img_src</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">_IMG_SRC_DEFAULT_URL_PREFIX</span> <span class="ow">in</span> <span class="n">img_src</span><span class="o">.</span><span class="n">split</span><span class="p">()[</span><span class="mi">0</span><span class="p">]:</span>
|
||||||
|
<span class="n">img_src_name</span> <span class="o">=</span> <span class="n">unquote</span><span class="p">(</span><span class="n">img_src</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">_IMG_SRC_DEFAULT_URL_PREFIX</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"?"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"%20"</span><span class="p">,</span> <span class="s2">"_"</span><span class="p">))</span>
|
||||||
|
<span class="n">img_src_name_first</span> <span class="o">=</span> <span class="n">img_src_name</span>
|
||||||
|
<span class="n">img_src_name_second</span> <span class="o">=</span> <span class="n">img_src_name</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s2">".svg"</span> <span class="ow">in</span> <span class="n">img_src_name</span><span class="o">.</span><span class="n">split</span><span class="p">()[</span><span class="mi">0</span><span class="p">]:</span>
|
||||||
|
<span class="n">img_src_name_second</span> <span class="o">=</span> <span class="n">img_src_name</span> <span class="o">+</span> <span class="s2">".png"</span>
|
||||||
|
|
||||||
|
<span class="n">img_src_size</span> <span class="o">=</span> <span class="n">img_src</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">_IMG_SRC_DEFAULT_URL_PREFIX</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"?"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">img_src_size</span> <span class="o">=</span> <span class="n">img_src_size</span><span class="p">[</span><span class="n">img_src_size</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">"="</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span> <span class="p">:</span> <span class="n">img_src_size</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s2">"&"</span><span class="p">)]</span>
|
||||||
|
<span class="n">img_src_name_md5</span> <span class="o">=</span> <span class="n">md5</span><span class="p">(</span><span class="n">img_src_name</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">))</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||||
|
<span class="n">img_src</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="n">_IMG_SRC_NEW_URL_PREFIX</span>
|
||||||
|
<span class="o">+</span> <span class="n">img_src_name_md5</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"/"</span>
|
||||||
|
<span class="o">+</span> <span class="n">img_src_name_md5</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="mi">2</span><span class="p">]</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"/"</span>
|
||||||
|
<span class="o">+</span> <span class="n">img_src_name_first</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"/"</span>
|
||||||
|
<span class="o">+</span> <span class="n">img_src_size</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"px-"</span>
|
||||||
|
<span class="o">+</span> <span class="n">img_src_name_second</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'get_thumbnail() redirected: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">img_src</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">img_src</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_results</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">,</span> <span class="n">attributes</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">infobox_title</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'itemLabel'</span><span class="p">)</span>
|
||||||
|
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="p">[</span><span class="s1">'item'</span><span class="p">]</span>
|
||||||
|
<span class="n">infobox_id_lang</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">infobox_urls</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">infobox_attributes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">infobox_content</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'itemDescription'</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="n">img_src</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">img_src_priority</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">attribute</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">:</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_str</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">value</span> <span class="o">!=</span> <span class="s1">''</span><span class="p">:</span>
|
||||||
|
<span class="n">attribute_type</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">attribute</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">attribute_type</span> <span class="ow">in</span> <span class="p">(</span><span class="n">WDURLAttribute</span><span class="p">,</span> <span class="n">WDArticle</span><span class="p">):</span>
|
||||||
|
<span class="c1"># get_select() method : there is group_concat(distinct ...;separator=", ")</span>
|
||||||
|
<span class="c1"># split the value here</span>
|
||||||
|
<span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">', '</span><span class="p">):</span>
|
||||||
|
<span class="n">infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_label</span><span class="p">(</span><span class="n">language</span><span class="p">),</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">attribute</span><span class="o">.</span><span class="n">kwargs</span><span class="p">})</span>
|
||||||
|
<span class="c1"># "normal" results (not infobox) include official website and Wikipedia links.</span>
|
||||||
|
<span class="k">if</span> <span class="s2">"list"</span> <span class="ow">in</span> <span class="n">display_type</span> <span class="ow">and</span> <span class="p">(</span><span class="n">attribute</span><span class="o">.</span><span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'official'</span><span class="p">)</span> <span class="ow">or</span> <span class="n">attribute_type</span> <span class="o">==</span> <span class="n">WDArticle</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="n">infobox_title</span><span class="p">,</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s2">"content"</span><span class="p">:</span> <span class="n">infobox_content</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="c1"># update the infobox_id with the wikipedia URL</span>
|
||||||
|
<span class="c1"># first the local wikipedia URL, and as fallback the english wikipedia URL</span>
|
||||||
|
<span class="k">if</span> <span class="n">attribute_type</span> <span class="o">==</span> <span class="n">WDArticle</span> <span class="ow">and</span> <span class="p">(</span>
|
||||||
|
<span class="p">(</span><span class="n">attribute</span><span class="o">.</span><span class="n">language</span> <span class="o">==</span> <span class="s1">'en'</span> <span class="ow">and</span> <span class="n">infobox_id_lang</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">or</span> <span class="n">attribute</span><span class="o">.</span><span class="n">language</span> <span class="o">!=</span> <span class="s1">'en'</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="n">infobox_id_lang</span> <span class="o">=</span> <span class="n">attribute</span><span class="o">.</span><span class="n">language</span>
|
||||||
|
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">url</span>
|
||||||
|
<span class="k">elif</span> <span class="n">attribute_type</span> <span class="o">==</span> <span class="n">WDImageAttribute</span><span class="p">:</span>
|
||||||
|
<span class="c1"># this attribute is an image.</span>
|
||||||
|
<span class="c1"># replace the current image only the priority is lower</span>
|
||||||
|
<span class="c1"># (the infobox contain only one image).</span>
|
||||||
|
<span class="k">if</span> <span class="n">attribute</span><span class="o">.</span><span class="n">priority</span> <span class="o">></span> <span class="n">img_src_priority</span><span class="p">:</span>
|
||||||
|
<span class="n">img_src</span> <span class="o">=</span> <span class="n">get_thumbnail</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="n">img_src_priority</span> <span class="o">=</span> <span class="n">attribute</span><span class="o">.</span><span class="n">priority</span>
|
||||||
|
<span class="k">elif</span> <span class="n">attribute_type</span> <span class="o">==</span> <span class="n">WDGeoAttribute</span><span class="p">:</span>
|
||||||
|
<span class="c1"># geocoordinate link</span>
|
||||||
|
<span class="c1"># use the area to get the OSM zoom</span>
|
||||||
|
<span class="c1"># Note: ignore the unit (must be km² otherwise the calculation is wrong)</span>
|
||||||
|
<span class="c1"># Should use normalized value p:P2046/psn:P2046/wikibase:quantityAmount</span>
|
||||||
|
<span class="n">area</span> <span class="o">=</span> <span class="n">attribute_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'P2046'</span><span class="p">)</span>
|
||||||
|
<span class="n">osm_zoom</span> <span class="o">=</span> <span class="n">area_to_osm_zoom</span><span class="p">(</span><span class="n">area</span><span class="p">)</span> <span class="k">if</span> <span class="n">area</span> <span class="k">else</span> <span class="mi">19</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_geo_url</span><span class="p">(</span><span class="n">attribute_result</span><span class="p">,</span> <span class="n">osm_zoom</span><span class="o">=</span><span class="n">osm_zoom</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">url</span><span class="p">:</span>
|
||||||
|
<span class="n">infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_label</span><span class="p">(</span><span class="n">language</span><span class="p">),</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'entity'</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">name</span><span class="p">})</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">infobox_attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span><span class="s1">'label'</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">get_label</span><span class="p">(</span><span class="n">language</span><span class="p">),</span> <span class="s1">'value'</span><span class="p">:</span> <span class="n">value</span><span class="p">,</span> <span class="s1">'entity'</span><span class="p">:</span> <span class="n">attribute</span><span class="o">.</span><span class="n">name</span><span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">infobox_id</span><span class="p">:</span>
|
||||||
|
<span class="n">infobox_id</span> <span class="o">=</span> <span class="n">replace_http_by_https</span><span class="p">(</span><span class="n">infobox_id</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># add the wikidata URL at the end</span>
|
||||||
|
<span class="n">infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'title'</span><span class="p">:</span> <span class="s1">'Wikidata'</span><span class="p">,</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">attribute_result</span><span class="p">[</span><span class="s1">'item'</span><span class="p">]})</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="p">(</span>
|
||||||
|
<span class="s2">"list"</span> <span class="ow">in</span> <span class="n">display_type</span>
|
||||||
|
<span class="ow">and</span> <span class="n">img_src</span> <span class="ow">is</span> <span class="kc">None</span>
|
||||||
|
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">infobox_attributes</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span>
|
||||||
|
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">infobox_urls</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span>
|
||||||
|
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">infobox_content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">infobox_urls</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s1">'url'</span><span class="p">],</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">infobox_title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">infobox_content</span><span class="p">})</span>
|
||||||
|
<span class="k">elif</span> <span class="s2">"infobox"</span> <span class="ow">in</span> <span class="n">display_type</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'infobox'</span><span class="p">:</span> <span class="n">infobox_title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'id'</span><span class="p">:</span> <span class="n">infobox_id</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">infobox_content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">img_src</span><span class="p">,</span>
|
||||||
|
<span class="s1">'urls'</span><span class="p">:</span> <span class="n">infobox_urls</span><span class="p">,</span>
|
||||||
|
<span class="s1">'attributes'</span><span class="p">:</span> <span class="n">infobox_attributes</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="n">attributes</span> <span class="o">=</span> <span class="n">get_attributes</span><span class="p">(</span><span class="n">language</span><span class="p">)</span>
|
||||||
|
<span class="n">select</span> <span class="o">=</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">]</span>
|
||||||
|
<span class="n">where</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">s</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">get_where</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">]))</span>
|
||||||
|
<span class="n">wikibase_label</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">s</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">get_wikibase_label</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">]))</span>
|
||||||
|
<span class="n">group_by</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">s</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">s</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="n">a</span><span class="o">.</span><span class="n">get_group_by</span><span class="p">()</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">attributes</span><span class="p">]))</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="n">QUERY_TEMPLATE</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%QUERY%'</span><span class="p">,</span> <span class="n">sparql_string_escape</span><span class="p">(</span><span class="n">query</span><span class="p">))</span>
|
||||||
|
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%SELECT%'</span><span class="p">,</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">select</span><span class="p">))</span>
|
||||||
|
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%WHERE%'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\n</span><span class="s1"> '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">where</span><span class="p">))</span>
|
||||||
|
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%WIKIBASE_LABELS%'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\n</span><span class="s1"> '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">wikibase_label</span><span class="p">))</span>
|
||||||
|
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">%G</span><span class="s1">ROUP_BY%'</span><span class="p">,</span> <span class="s1">' '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">group_by</span><span class="p">))</span>
|
||||||
|
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%LANGUAGE%'</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">query</span><span class="p">,</span> <span class="n">attributes</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_attributes</span><span class="p">(</span><span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-statements</span>
|
||||||
|
<span class="n">attributes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_value</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_amount</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDAmountAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_label</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDLabelAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_url</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDURLAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_image</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDImageAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="p">,</span> <span class="n">priority</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">add_date</span><span class="p">(</span><span class="n">name</span><span class="p">):</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDDateAttribute</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># Dates</span>
|
||||||
|
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'P571'</span><span class="p">,</span> <span class="c1"># inception date</span>
|
||||||
|
<span class="s1">'P576'</span><span class="p">,</span> <span class="c1"># dissolution date</span>
|
||||||
|
<span class="s1">'P580'</span><span class="p">,</span> <span class="c1"># start date</span>
|
||||||
|
<span class="s1">'P582'</span><span class="p">,</span> <span class="c1"># end date</span>
|
||||||
|
<span class="s1">'P569'</span><span class="p">,</span> <span class="c1"># date of birth</span>
|
||||||
|
<span class="s1">'P570'</span><span class="p">,</span> <span class="c1"># date of death</span>
|
||||||
|
<span class="s1">'P619'</span><span class="p">,</span> <span class="c1"># date of spacecraft launch</span>
|
||||||
|
<span class="s1">'P620'</span><span class="p">,</span>
|
||||||
|
<span class="p">]:</span> <span class="c1"># date of spacecraft landing</span>
|
||||||
|
<span class="n">add_date</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'P27'</span><span class="p">,</span> <span class="c1"># country of citizenship</span>
|
||||||
|
<span class="s1">'P495'</span><span class="p">,</span> <span class="c1"># country of origin</span>
|
||||||
|
<span class="s1">'P17'</span><span class="p">,</span> <span class="c1"># country</span>
|
||||||
|
<span class="s1">'P159'</span><span class="p">,</span>
|
||||||
|
<span class="p">]:</span> <span class="c1"># headquarters location</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Places</span>
|
||||||
|
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'P36'</span><span class="p">,</span> <span class="c1"># capital</span>
|
||||||
|
<span class="s1">'P35'</span><span class="p">,</span> <span class="c1"># head of state</span>
|
||||||
|
<span class="s1">'P6'</span><span class="p">,</span> <span class="c1"># head of government</span>
|
||||||
|
<span class="s1">'P122'</span><span class="p">,</span> <span class="c1"># basic form of government</span>
|
||||||
|
<span class="s1">'P37'</span><span class="p">,</span>
|
||||||
|
<span class="p">]:</span> <span class="c1"># official language</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P1082'</span><span class="p">)</span> <span class="c1"># population</span>
|
||||||
|
<span class="n">add_amount</span><span class="p">(</span><span class="s1">'P2046'</span><span class="p">)</span> <span class="c1"># area</span>
|
||||||
|
<span class="n">add_amount</span><span class="p">(</span><span class="s1">'P281'</span><span class="p">)</span> <span class="c1"># postal code</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P38'</span><span class="p">)</span> <span class="c1"># currency</span>
|
||||||
|
<span class="n">add_amount</span><span class="p">(</span><span class="s1">'P2048'</span><span class="p">)</span> <span class="c1"># height (building)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Media</span>
|
||||||
|
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'P400'</span><span class="p">,</span> <span class="c1"># platform (videogames, computing)</span>
|
||||||
|
<span class="s1">'P50'</span><span class="p">,</span> <span class="c1"># author</span>
|
||||||
|
<span class="s1">'P170'</span><span class="p">,</span> <span class="c1"># creator</span>
|
||||||
|
<span class="s1">'P57'</span><span class="p">,</span> <span class="c1"># director</span>
|
||||||
|
<span class="s1">'P175'</span><span class="p">,</span> <span class="c1"># performer</span>
|
||||||
|
<span class="s1">'P178'</span><span class="p">,</span> <span class="c1"># developer</span>
|
||||||
|
<span class="s1">'P162'</span><span class="p">,</span> <span class="c1"># producer</span>
|
||||||
|
<span class="s1">'P176'</span><span class="p">,</span> <span class="c1"># manufacturer</span>
|
||||||
|
<span class="s1">'P58'</span><span class="p">,</span> <span class="c1"># screenwriter</span>
|
||||||
|
<span class="s1">'P272'</span><span class="p">,</span> <span class="c1"># production company</span>
|
||||||
|
<span class="s1">'P264'</span><span class="p">,</span> <span class="c1"># record label</span>
|
||||||
|
<span class="s1">'P123'</span><span class="p">,</span> <span class="c1"># publisher</span>
|
||||||
|
<span class="s1">'P449'</span><span class="p">,</span> <span class="c1"># original network</span>
|
||||||
|
<span class="s1">'P750'</span><span class="p">,</span> <span class="c1"># distributed by</span>
|
||||||
|
<span class="s1">'P86'</span><span class="p">,</span>
|
||||||
|
<span class="p">]:</span> <span class="c1"># composer</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">add_date</span><span class="p">(</span><span class="s1">'P577'</span><span class="p">)</span> <span class="c1"># publication date</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P136'</span><span class="p">)</span> <span class="c1"># genre (music, film, artistic...)</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P364'</span><span class="p">)</span> <span class="c1"># original language</span>
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P212'</span><span class="p">)</span> <span class="c1"># ISBN-13</span>
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P957'</span><span class="p">)</span> <span class="c1"># ISBN-10</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P275'</span><span class="p">)</span> <span class="c1"># copyright license</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P277'</span><span class="p">)</span> <span class="c1"># programming language</span>
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P348'</span><span class="p">)</span> <span class="c1"># version</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P840'</span><span class="p">)</span> <span class="c1"># narrative location</span>
|
||||||
|
|
||||||
|
<span class="c1"># Languages</span>
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P1098'</span><span class="p">)</span> <span class="c1"># number of speakers</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P282'</span><span class="p">)</span> <span class="c1"># writing system</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P1018'</span><span class="p">)</span> <span class="c1"># language regulatory body</span>
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P218'</span><span class="p">)</span> <span class="c1"># language code (ISO 639-1)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Other</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P169'</span><span class="p">)</span> <span class="c1"># ceo</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P112'</span><span class="p">)</span> <span class="c1"># founded by</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P1454'</span><span class="p">)</span> <span class="c1"># legal form (company, organization)</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P137'</span><span class="p">)</span> <span class="c1"># operator (service, facility, ...)</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P1029'</span><span class="p">)</span> <span class="c1"># crew members (tripulation)</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P225'</span><span class="p">)</span> <span class="c1"># taxon name</span>
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P274'</span><span class="p">)</span> <span class="c1"># chemical formula</span>
|
||||||
|
<span class="n">add_label</span><span class="p">(</span><span class="s1">'P1346'</span><span class="p">)</span> <span class="c1"># winner (sports, contests, ...)</span>
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P1120'</span><span class="p">)</span> <span class="c1"># number of deaths</span>
|
||||||
|
<span class="n">add_value</span><span class="p">(</span><span class="s1">'P498'</span><span class="p">)</span> <span class="c1"># currency code (ISO 4217)</span>
|
||||||
|
|
||||||
|
<span class="c1"># URL</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P856'</span><span class="p">,</span> <span class="n">official</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="c1"># official website</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDArticle</span><span class="p">(</span><span class="n">language</span><span class="p">))</span> <span class="c1"># wikipedia (user language)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">language</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'en'</span><span class="p">):</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDArticle</span><span class="p">(</span><span class="s1">'en'</span><span class="p">))</span> <span class="c1"># wikipedia (english)</span>
|
||||||
|
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P1324'</span><span class="p">)</span> <span class="c1"># source code repository</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P1581'</span><span class="p">)</span> <span class="c1"># blog</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P434'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'musicbrainz_artist'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P435'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'musicbrainz_work'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P436'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'musicbrainz_release_group'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P966'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'musicbrainz_label'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P345'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'imdb_id'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P2397'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'youtube_channel'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P1651'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'youtube_video'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P2002'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'twitter_profile'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P2013'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'facebook_profile'</span><span class="p">)</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P2003'</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'instagram_profile'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Fediverse</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P4033'</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="s1">'/@'</span><span class="p">)</span> <span class="c1"># Mastodon user</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P11947'</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="s1">'/c/'</span><span class="p">)</span> <span class="c1"># Lemmy community</span>
|
||||||
|
<span class="n">add_url</span><span class="p">(</span><span class="s1">'P12622'</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="s1">'/c/'</span><span class="p">)</span> <span class="c1"># PeerTube channel</span>
|
||||||
|
|
||||||
|
<span class="c1"># Map</span>
|
||||||
|
<span class="n">attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">WDGeoAttribute</span><span class="p">(</span><span class="s1">'P625'</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># Image</span>
|
||||||
|
<span class="n">add_image</span><span class="p">(</span><span class="s1">'P15'</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'wikimedia_image'</span><span class="p">)</span> <span class="c1"># route map</span>
|
||||||
|
<span class="n">add_image</span><span class="p">(</span><span class="s1">'P242'</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'wikimedia_image'</span><span class="p">)</span> <span class="c1"># locator map</span>
|
||||||
|
<span class="n">add_image</span><span class="p">(</span><span class="s1">'P154'</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'wikimedia_image'</span><span class="p">)</span> <span class="c1"># logo</span>
|
||||||
|
<span class="n">add_image</span><span class="p">(</span><span class="s1">'P18'</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'wikimedia_image'</span><span class="p">)</span> <span class="c1"># image</span>
|
||||||
|
<span class="n">add_image</span><span class="p">(</span><span class="s1">'P41'</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'wikimedia_image'</span><span class="p">)</span> <span class="c1"># flag</span>
|
||||||
|
<span class="n">add_image</span><span class="p">(</span><span class="s1">'P2716'</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">6</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'wikimedia_image'</span><span class="p">)</span> <span class="c1"># collage</span>
|
||||||
|
<span class="n">add_image</span><span class="p">(</span><span class="s1">'P2910'</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">7</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="s1">'wikimedia_image'</span><span class="p">)</span> <span class="c1"># icon</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">attributes</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WDAttribute</span><span class="p">:</span>
|
||||||
|
<span class="vm">__slots__</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'name'</span><span class="p">,)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'(group_concat(distinct ?</span><span class="si">{name}</span><span class="s1">;separator=", ") as ?</span><span class="si">{name}</span><span class="s1">s)'</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_label</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">get_label_for_entity</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"OPTIONAL { ?item wdt:</span><span class="si">{name}</span><span class="s2"> ?</span><span class="si">{name}</span><span class="s2"> . }"</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_wikibase_label</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="k">return</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'s'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'<'</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span> <span class="o">+</span> <span class="s1">':'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'>'</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WDAmountAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'?</span><span class="si">{name}</span><span class="s1"> ?</span><span class="si">{name}</span><span class="s1">Unit'</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""" OPTIONAL { ?item p:</span><span class="si">{name}</span><span class="s2"> ?</span><span class="si">{name}</span><span class="s2">Node .</span>
|
||||||
|
<span class="s2"> ?</span><span class="si">{name}</span><span class="s2">Node rdf:type wikibase:BestRank ; ps:</span><span class="si">{name}</span><span class="s2"> ?</span><span class="si">{name}</span><span class="s2"> .</span>
|
||||||
|
<span class="s2"> OPTIONAL { ?</span><span class="si">{name}</span><span class="s2">Node psv:</span><span class="si">{name}</span><span class="s2">/wikibase:quantityUnit ?</span><span class="si">{name}</span><span class="s2">Unit. } }"""</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
|
||||||
|
<span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="n">unit</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s2">"Unit"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">unit</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">unit</span> <span class="o">=</span> <span class="n">unit</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'http://www.wikidata.org/entity/'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">value</span> <span class="o">+</span> <span class="s2">" "</span> <span class="o">+</span> <span class="n">get_label_for_entity</span><span class="p">(</span><span class="n">unit</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">value</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WDArticle</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="vm">__slots__</span> <span class="o">=</span> <span class="s1">'language'</span><span class="p">,</span> <span class="s1">'kwargs'</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">language</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="s1">'wikipedia'</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">language</span> <span class="o">=</span> <span class="n">language</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span> <span class="ow">or</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_label</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="c1"># language parameter is ignored</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"Wikipedia (</span><span class="si">{language}</span><span class="s2">)"</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{language}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">language</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"?article</span><span class="si">{language}</span><span class="s2"> ?articleName</span><span class="si">{language}</span><span class="s2">"</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{language}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">language</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"""OPTIONAL { ?article</span><span class="si">{language}</span><span class="s2"> schema:about ?item ;</span>
|
||||||
|
<span class="s2"> schema:inLanguage "</span><span class="si">{language}</span><span class="s2">" ;</span>
|
||||||
|
<span class="s2"> schema:isPartOf <https://</span><span class="si">{language}</span><span class="s2">.wikipedia.org/> ;</span>
|
||||||
|
<span class="s2"> schema:name ?articleName</span><span class="si">{language}</span><span class="s2"> . }"""</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
|
||||||
|
<span class="s1">'</span><span class="si">{language}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">language</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="n">key</span> <span class="o">=</span> <span class="s1">'article</span><span class="si">{language}</span><span class="s1">'</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{language}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">language</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">key</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WDLabelAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'(group_concat(distinct ?</span><span class="si">{name}</span><span class="s1">Label;separator=", ") as ?</span><span class="si">{name}</span><span class="s1">Labels)'</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"OPTIONAL { ?item wdt:</span><span class="si">{name}</span><span class="s2"> ?</span><span class="si">{name}</span><span class="s2"> . }"</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_wikibase_label</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"?</span><span class="si">{name}</span><span class="s2"> rdfs:label ?</span><span class="si">{name}</span><span class="s2">Label ."</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'Labels'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WDURLAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">HTTP_WIKIMEDIA_IMAGE</span> <span class="o">=</span> <span class="s1">'http://commons.wikimedia.org/wiki/Special:FilePath/'</span>
|
||||||
|
|
||||||
|
<span class="vm">__slots__</span> <span class="o">=</span> <span class="s1">'url_id'</span><span class="p">,</span> <span class="s1">'url_path_prefix'</span><span class="p">,</span> <span class="s1">'kwargs'</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">url_path_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""</span>
|
||||||
|
<span class="sd"> :param url_id: ID matching one key in ``external_urls.json`` for</span>
|
||||||
|
<span class="sd"> converting IDs to full URLs.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param url_path_prefix: Path prefix if the values are of format</span>
|
||||||
|
<span class="sd"> ``account@domain``. If provided, value are rewritten to</span>
|
||||||
|
<span class="sd"> ``https://<domain><url_path_prefix><account>``. For example::</span>
|
||||||
|
|
||||||
|
<span class="sd"> WDURLAttribute('P4033', url_path_prefix='/@')</span>
|
||||||
|
|
||||||
|
<span class="sd"> Adds Property `P4033 <https://www.wikidata.org/wiki/Property:P4033>`_</span>
|
||||||
|
<span class="sd"> to the wikidata query. This field might return for example</span>
|
||||||
|
<span class="sd"> ``libreoffice@fosstodon.org`` and the URL built from this is then:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - account: ``libreoffice``</span>
|
||||||
|
<span class="sd"> - domain: ``fosstodon.org``</span>
|
||||||
|
<span class="sd"> - result url: https://fosstodon.org/@libreoffice</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">url_id</span> <span class="o">=</span> <span class="n">url_id</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">url_path_prefix</span> <span class="o">=</span> <span class="n">url_path_prefix</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'s'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">value</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">','</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">url_id</span><span class="p">:</span>
|
||||||
|
<span class="n">url_id</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">url_id</span>
|
||||||
|
<span class="k">if</span> <span class="n">value</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">WDURLAttribute</span><span class="o">.</span><span class="n">HTTP_WIKIMEDIA_IMAGE</span><span class="p">):</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">value</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">WDURLAttribute</span><span class="o">.</span><span class="n">HTTP_WIKIMEDIA_IMAGE</span><span class="p">)</span> <span class="p">:]</span>
|
||||||
|
<span class="n">url_id</span> <span class="o">=</span> <span class="s1">'wikimedia_image'</span>
|
||||||
|
<span class="k">return</span> <span class="n">get_external_url</span><span class="p">(</span><span class="n">url_id</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">url_path_prefix</span><span class="p">:</span>
|
||||||
|
<span class="p">[</span><span class="n">account</span><span class="p">,</span> <span class="n">domain</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">x</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"@ "</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">value</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">'@'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)]</span>
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"https://</span><span class="si">{</span><span class="n">domain</span><span class="si">}{</span><span class="bp">self</span><span class="o">.</span><span class="n">url_path_prefix</span><span class="si">}{</span><span class="n">account</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">value</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WDGeoAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_label</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"OpenStreetMap"</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"?</span><span class="si">{name}</span><span class="s2">Lat ?</span><span class="si">{name}</span><span class="s2">Long"</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"""OPTIONAL { ?item p:</span><span class="si">{name}</span><span class="s2">/psv:</span><span class="si">{name}</span><span class="s2"> [</span>
|
||||||
|
<span class="s2"> wikibase:geoLatitude ?</span><span class="si">{name}</span><span class="s2">Lat ;</span>
|
||||||
|
<span class="s2"> wikibase:geoLongitude ?</span><span class="si">{name}</span><span class="s2">Long ] }"""</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
|
||||||
|
<span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="n">latitude</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'Lat'</span><span class="p">)</span>
|
||||||
|
<span class="n">longitude</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'Long'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">latitude</span> <span class="ow">and</span> <span class="n">longitude</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">latitude</span> <span class="o">+</span> <span class="s1">' '</span> <span class="o">+</span> <span class="n">longitude</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_geo_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">osm_zoom</span><span class="o">=</span><span class="mi">19</span><span class="p">):</span>
|
||||||
|
<span class="n">latitude</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'Lat'</span><span class="p">)</span>
|
||||||
|
<span class="n">longitude</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'Long'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">latitude</span> <span class="ow">and</span> <span class="n">longitude</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">get_earth_coordinates_url</span><span class="p">(</span><span class="n">latitude</span><span class="p">,</span> <span class="n">longitude</span><span class="p">,</span> <span class="n">osm_zoom</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WDImageAttribute</span><span class="p">(</span><span class="n">WDURLAttribute</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="vm">__slots__</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'priority'</span><span class="p">,)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">priority</span><span class="o">=</span><span class="mi">100</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url_id</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="n">priority</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WDDateAttribute</span><span class="p">(</span><span class="n">WDAttribute</span><span class="p">):</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_select</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'?</span><span class="si">{name}</span><span class="s1"> ?</span><span class="si">{name}</span><span class="s1">timePrecision ?</span><span class="si">{name}</span><span class="s1">timeZone ?</span><span class="si">{name}</span><span class="s1">timeCalendar'</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_where</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="c1"># To remove duplicate, add</span>
|
||||||
|
<span class="c1"># FILTER NOT EXISTS { ?item p:{name}/psv:{name}/wikibase:timeValue ?{name}bis FILTER (?{name}bis < ?{name}) }</span>
|
||||||
|
<span class="c1"># this filter is too slow, so the response function ignore duplicate results</span>
|
||||||
|
<span class="c1"># (see the seen_entities variable)</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"""OPTIONAL { ?item p:</span><span class="si">{name}</span><span class="s2">/psv:</span><span class="si">{name}</span><span class="s2"> [</span>
|
||||||
|
<span class="s2"> wikibase:timeValue ?</span><span class="si">{name}</span><span class="s2"> ;</span>
|
||||||
|
<span class="s2"> wikibase:timePrecision ?</span><span class="si">{name}</span><span class="s2">timePrecision ;</span>
|
||||||
|
<span class="s2"> wikibase:timeTimezone ?</span><span class="si">{name}</span><span class="s2">timeZone ;</span>
|
||||||
|
<span class="s2"> wikibase:timeCalendarModel ?</span><span class="si">{name}</span><span class="s2">timeCalendar ] . }</span>
|
||||||
|
<span class="s2"> hint:Prior hint:rangeSafe true;"""</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span>
|
||||||
|
<span class="s1">'</span><span class="si">{name}</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_group_by</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_select</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">format_8</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="c1"># precision: less than a year</span>
|
||||||
|
<span class="k">return</span> <span class="n">value</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">format_9</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
|
||||||
|
<span class="n">year</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="c1"># precision: year</span>
|
||||||
|
<span class="k">if</span> <span class="n">year</span> <span class="o"><</span> <span class="mi">1584</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">year</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">year</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">year</span><span class="p">)</span>
|
||||||
|
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">format_date</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">'yyyy'</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">format_10</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
|
||||||
|
<span class="c1"># precision: month</span>
|
||||||
|
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">format_date</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">'MMMM y'</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">format_11</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
|
||||||
|
<span class="c1"># precision: day</span>
|
||||||
|
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">format_date</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">'full'</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">format_13</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
|
||||||
|
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
|
||||||
|
<span class="c1"># precision: minute</span>
|
||||||
|
<span class="k">return</span> <span class="p">(</span>
|
||||||
|
<span class="n">get_datetime_format</span><span class="p">(</span><span class="nb">format</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{0}</span><span class="s1">'</span><span class="p">,</span> <span class="n">format_time</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="s1">'full'</span><span class="p">,</span> <span class="n">tzinfo</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">))</span>
|
||||||
|
<span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="si">{1}</span><span class="s1">'</span><span class="p">,</span> <span class="n">format_date</span><span class="p">(</span><span class="n">timestamp</span><span class="p">,</span> <span class="s1">'short'</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">))</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">format_14</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="p">):</span>
|
||||||
|
<span class="c1"># precision: second.</span>
|
||||||
|
<span class="k">return</span> <span class="n">format_datetime</span><span class="p">(</span><span class="n">isoparse</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="nb">format</span><span class="o">=</span><span class="s1">'full'</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">DATE_FORMAT</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'0'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">1000000000</span><span class="p">),</span>
|
||||||
|
<span class="s1">'1'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">100000000</span><span class="p">),</span>
|
||||||
|
<span class="s1">'2'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">10000000</span><span class="p">),</span>
|
||||||
|
<span class="s1">'3'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">1000000</span><span class="p">),</span>
|
||||||
|
<span class="s1">'4'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">100000</span><span class="p">),</span>
|
||||||
|
<span class="s1">'5'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">10000</span><span class="p">),</span>
|
||||||
|
<span class="s1">'6'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">1000</span><span class="p">),</span>
|
||||||
|
<span class="s1">'7'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">100</span><span class="p">),</span>
|
||||||
|
<span class="s1">'8'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_8'</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span>
|
||||||
|
<span class="s1">'9'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_9'</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="c1"># year</span>
|
||||||
|
<span class="s1">'10'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_10'</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="c1"># month</span>
|
||||||
|
<span class="s1">'11'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_11'</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># day</span>
|
||||||
|
<span class="s1">'12'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_13'</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># hour (not supported by babel, display minute)</span>
|
||||||
|
<span class="s1">'13'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_13'</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># minute</span>
|
||||||
|
<span class="s1">'14'</span><span class="p">:</span> <span class="p">(</span><span class="s1">'format_14'</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> <span class="c1"># second</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_str</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">,</span> <span class="n">language</span><span class="p">):</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">value</span> <span class="o">==</span> <span class="s1">''</span> <span class="ow">or</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
<span class="n">precision</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s1">'timePrecision'</span><span class="p">)</span>
|
||||||
|
<span class="n">date_format</span> <span class="o">=</span> <span class="n">WDDateAttribute</span><span class="o">.</span><span class="n">DATE_FORMAT</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">precision</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">date_format</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">format_method</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">date_format</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="n">precision</span> <span class="o">=</span> <span class="n">date_format</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">precision</span> <span class="o">>=</span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">t</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">value</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'-'</span><span class="p">):</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">t</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">t</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="k">return</span> <span class="n">format_method</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">language</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||||
|
<span class="k">return</span> <span class="n">value</span>
|
||||||
|
<span class="k">return</span> <span class="n">value</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">debug_explain_wikidata_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s1">'GET'</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">method</span> <span class="o">==</span> <span class="s1">'GET'</span><span class="p">:</span>
|
||||||
|
<span class="n">http_response</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">SPARQL_EXPLAIN_URL</span> <span class="o">+</span> <span class="s1">'&'</span> <span class="o">+</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">}),</span> <span class="n">headers</span><span class="o">=</span><span class="n">get_headers</span><span class="p">())</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">http_response</span> <span class="o">=</span> <span class="n">post</span><span class="p">(</span><span class="n">SPARQL_EXPLAIN_URL</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="p">{</span><span class="s1">'query'</span><span class="p">:</span> <span class="n">query</span><span class="p">},</span> <span class="n">headers</span><span class="o">=</span><span class="n">get_headers</span><span class="p">())</span>
|
||||||
|
<span class="n">http_response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||||
|
<span class="k">return</span> <span class="n">http_response</span><span class="o">.</span><span class="n">content</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">engine_settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="c1"># WIKIDATA_PROPERTIES : add unit symbols</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">WIKIDATA_UNITS</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">WIKIDATA_PROPERTIES</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span><span class="p">[</span><span class="s1">'symbol'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># WIKIDATA_PROPERTIES : add property labels</span>
|
||||||
|
<span class="n">wikidata_property_names</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">attribute</span> <span class="ow">in</span> <span class="n">get_attributes</span><span class="p">(</span><span class="s1">'en'</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">attribute</span><span class="p">)</span> <span class="ow">in</span> <span class="p">(</span><span class="n">WDAttribute</span><span class="p">,</span> <span class="n">WDAmountAttribute</span><span class="p">,</span> <span class="n">WDURLAttribute</span><span class="p">,</span> <span class="n">WDDateAttribute</span><span class="p">,</span> <span class="n">WDLabelAttribute</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">attribute</span><span class="o">.</span><span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">WIKIDATA_PROPERTIES</span><span class="p">:</span>
|
||||||
|
<span class="n">wikidata_property_names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">"wd:"</span> <span class="o">+</span> <span class="n">attribute</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">QUERY_PROPERTY_NAMES</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'%ATTRIBUTES%'</span><span class="p">,</span> <span class="s2">" "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">wikidata_property_names</span><span class="p">))</span>
|
||||||
|
<span class="n">jsonresponse</span> <span class="o">=</span> <span class="n">send_wikidata_query</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">jsonresponse</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'results'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'bindings'</span><span class="p">,</span> <span class="p">{}):</span>
|
||||||
|
<span class="n">name</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">][</span><span class="s1">'value'</span><span class="p">]</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'name'</span><span class="p">][</span><span class="s1">'xml:lang'</span><span class="p">]</span>
|
||||||
|
<span class="n">entity_id</span> <span class="o">=</span> <span class="n">result</span><span class="p">[</span><span class="s1">'item'</span><span class="p">][</span><span class="s1">'value'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'http://www.wikidata.org/entity/'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="n">WIKIDATA_PROPERTIES</span><span class="p">[(</span><span class="n">entity_id</span><span class="p">,</span> <span class="n">lang</span><span class="p">)]</span> <span class="o">=</span> <span class="n">name</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikidata.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Uses languages evaluated from :py:obj:`wikipedia.fetch_wikimedia_traits</span>
|
||||||
|
<span class="sd"> <searx.engines.wikipedia.fetch_wikimedia_traits>` and removes</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``traits.custom['wiki_netloc']``: wikidata does not have net-locations for</span>
|
||||||
|
<span class="sd"> the languages and the list of all</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``traits.custom['WIKIPEDIA_LANGUAGES']``: not used in the wikipedia engine</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">fetch_wikimedia_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">)</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'WIKIPEDIA_LANGUAGES'</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
441
_modules/searx/engines/wikipedia.html
Normal file
@@ -0,0 +1,441 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.wikipedia — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.wikipedia</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.wikipedia</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This module implements the Wikipedia engine. Some of this implementations</span>
|
||||||
|
<span class="sd">are shared by other engines:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :ref:`wikidata engine`</span>
|
||||||
|
|
||||||
|
<span class="sd">The list of supported languages is :py:obj:`fetched <fetch_wikimedia_traits>` from</span>
|
||||||
|
<span class="sd">the article linked by :py:obj:`list_of_wikipedias`.</span>
|
||||||
|
|
||||||
|
<span class="sd">Unlike traditional search engines, wikipedia does not support one Wikipedia for</span>
|
||||||
|
<span class="sd">all languages, but there is one Wikipedia for each supported language. Some of</span>
|
||||||
|
<span class="sd">these Wikipedias have a LanguageConverter_ enabled</span>
|
||||||
|
<span class="sd">(:py:obj:`rest_v1_summary_url`).</span>
|
||||||
|
|
||||||
|
<span class="sd">A LanguageConverter_ (LC) is a system based on language variants that</span>
|
||||||
|
<span class="sd">automatically converts the content of a page into a different variant. A variant</span>
|
||||||
|
<span class="sd">is mostly the same language in a different script.</span>
|
||||||
|
|
||||||
|
<span class="sd">- `Wikipedias in multiple writing systems`_</span>
|
||||||
|
<span class="sd">- `Automatic conversion between traditional and simplified Chinese characters`_</span>
|
||||||
|
|
||||||
|
<span class="sd">PR-2554_:</span>
|
||||||
|
<span class="sd"> The Wikipedia link returned by the API is still the same in all cases</span>
|
||||||
|
<span class="sd"> (`https://zh.wikipedia.org/wiki/出租車`_) but if your browser's</span>
|
||||||
|
<span class="sd"> ``Accept-Language`` is set to any of ``zh``, ``zh-CN``, ``zh-TW``, ``zh-HK``</span>
|
||||||
|
<span class="sd"> or .. Wikipedia's LC automatically returns the desired script in their</span>
|
||||||
|
<span class="sd"> web-page.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - You can test the API here: https://reqbin.com/gesg2kvx</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _https://zh.wikipedia.org/wiki/出租車:</span>
|
||||||
|
<span class="sd"> https://zh.wikipedia.org/wiki/%E5%87%BA%E7%A7%9F%E8%BB%8A</span>
|
||||||
|
|
||||||
|
<span class="sd">To support Wikipedia's LanguageConverter_, a SearXNG request to Wikipedia uses</span>
|
||||||
|
<span class="sd">:py:obj:`get_wiki_params` and :py:obj:`wiki_lc_locale_variants' in the</span>
|
||||||
|
<span class="sd">:py:obj:`fetch_wikimedia_traits` function.</span>
|
||||||
|
|
||||||
|
<span class="sd">To test in SearXNG, query for ``!wp 出租車`` with each of the available Chinese</span>
|
||||||
|
<span class="sd">options:</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``!wp 出租車 :zh`` should show 出租車</span>
|
||||||
|
<span class="sd">- ``!wp 出租車 :zh-CN`` should show 出租车</span>
|
||||||
|
<span class="sd">- ``!wp 出租車 :zh-TW`` should show 計程車</span>
|
||||||
|
<span class="sd">- ``!wp 出租車 :zh-HK`` should show 的士</span>
|
||||||
|
<span class="sd">- ``!wp 出租車 :zh-SG`` should show 德士</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _LanguageConverter:</span>
|
||||||
|
<span class="sd"> https://www.mediawiki.org/wiki/Writing_systems#LanguageConverter</span>
|
||||||
|
<span class="sd">.. _Wikipedias in multiple writing systems:</span>
|
||||||
|
<span class="sd"> https://meta.wikimedia.org/wiki/Wikipedias_in_multiple_writing_systems</span>
|
||||||
|
<span class="sd">.. _Automatic conversion between traditional and simplified Chinese characters:</span>
|
||||||
|
<span class="sd"> https://en.wikipedia.org/wiki/Chinese_Wikipedia#Automatic_conversion_between_traditional_and_simplified_Chinese_characters</span>
|
||||||
|
<span class="sd">.. _PR-2554: https://github.com/searx/searx/pull/2554</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib.parse</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">utils</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">network</span> <span class="k">as</span> <span class="n">_network</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">locales</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://www.wikipedia.org/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s1">'Q52'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://en.wikipedia.org/api/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'JSON'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">display_type</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"infobox"</span><span class="p">]</span>
|
||||||
|
<span class="sd">"""A list of display types composed from ``infobox`` and ``list``. The latter</span>
|
||||||
|
<span class="sd">one will add a hit to the result list. The first one will show a hit in the</span>
|
||||||
|
<span class="sd">info box. Both values can be set, or one of the two can be set."""</span>
|
||||||
|
|
||||||
|
<span class="n">send_accept_language_header</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="sd">"""The HTTP ``Accept-Language`` header is needed for wikis where</span>
|
||||||
|
<span class="sd">LanguageConverter_ is enabled."""</span>
|
||||||
|
|
||||||
|
<span class="n">list_of_wikipedias</span> <span class="o">=</span> <span class="s1">'https://meta.wikimedia.org/wiki/List_of_Wikipedias'</span>
|
||||||
|
<span class="sd">"""`List of all wikipedias <https://meta.wikimedia.org/wiki/List_of_Wikipedias>`_</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">wikipedia_article_depth</span> <span class="o">=</span> <span class="s1">'https://meta.wikimedia.org/wiki/Wikipedia_article_depth'</span>
|
||||||
|
<span class="sd">"""The *editing depth* of Wikipedia is one of several possible rough indicators</span>
|
||||||
|
<span class="sd">of the encyclopedia's collaborative quality, showing how frequently its articles</span>
|
||||||
|
<span class="sd">are updated. The measurement of depth was introduced after some limitations of</span>
|
||||||
|
<span class="sd">the classic measurement of article count were realized.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">rest_v1_summary_url</span> <span class="o">=</span> <span class="s1">'https://</span><span class="si">{wiki_netloc}</span><span class="s1">/api/rest_v1/page/summary/</span><span class="si">{title}</span><span class="s1">'</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">`wikipedia rest_v1 summary API`_:</span>
|
||||||
|
<span class="sd"> The summary response includes an extract of the first paragraph of the page in</span>
|
||||||
|
<span class="sd"> plain text and HTML as well as the type of page. This is useful for page</span>
|
||||||
|
<span class="sd"> previews (fka. Hovercards, aka. Popups) on the web and link previews in the</span>
|
||||||
|
<span class="sd"> apps.</span>
|
||||||
|
|
||||||
|
<span class="sd">HTTP ``Accept-Language`` header (:py:obj:`send_accept_language_header`):</span>
|
||||||
|
<span class="sd"> The desired language variant code for wikis where LanguageConverter_ is</span>
|
||||||
|
<span class="sd"> enabled.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _wikipedia rest_v1 summary API:</span>
|
||||||
|
<span class="sd"> https://en.wikipedia.org/api/rest_v1/#/Page%20content/get_page_summary__title_</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">wiki_lc_locale_variants</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"zh"</span><span class="p">:</span> <span class="p">(</span>
|
||||||
|
<span class="s2">"zh-CN"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"zh-HK"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"zh-MO"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"zh-MY"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"zh-SG"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"zh-TW"</span><span class="p">,</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="s2">"zh-classical"</span><span class="p">:</span> <span class="p">(</span><span class="s2">"zh-classical"</span><span class="p">,),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""Mapping rule of the LanguageConverter_ to map a language and its variants to</span>
|
||||||
|
<span class="sd">a Locale (used in the HTTP ``Accept-Language`` header). For example see `LC</span>
|
||||||
|
<span class="sd">Chinese`_.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _LC Chinese:</span>
|
||||||
|
<span class="sd"> https://meta.wikimedia.org/wiki/Wikipedias_in_multiple_writing_systems#Chinese</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">wikipedia_script_variants</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"zh"</span><span class="p">:</span> <span class="p">(</span>
|
||||||
|
<span class="s2">"zh_Hant"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"zh_Hans"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_wiki_params">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikipedia.get_wiki_params">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_wiki_params</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">eng_traits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns the Wikipedia language tag and the netloc that fits to the</span>
|
||||||
|
<span class="sd"> ``sxng_locale``. To support LanguageConverter_ this function rates a locale</span>
|
||||||
|
<span class="sd"> (region) higher than a language (compare :py:obj:`wiki_lc_locale_variants`).</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_region</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">sxng_locale</span><span class="p">,</span> <span class="s1">'en'</span><span class="p">))</span>
|
||||||
|
<span class="n">wiki_netloc</span> <span class="o">=</span> <span class="n">eng_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">,</span> <span class="s1">'en.wikipedia.org'</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">wiki_netloc</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikipedia.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Assemble a request (`wikipedia rest_v1 summary API`_)."""</span>
|
||||||
|
<span class="k">if</span> <span class="n">query</span><span class="o">.</span><span class="n">islower</span><span class="p">():</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">title</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">_eng_tag</span><span class="p">,</span> <span class="n">wiki_netloc</span> <span class="o">=</span> <span class="n">get_wiki_params</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'searxng_locale'</span><span class="p">],</span> <span class="n">traits</span><span class="p">)</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">rest_v1_summary_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">wiki_netloc</span><span class="o">=</span><span class="n">wiki_netloc</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="n">title</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'soft_max_redirects'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">2</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># get response from search-request</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">404</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="p">[]</span>
|
||||||
|
<span class="k">if</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">400</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">api_result</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="p">(</span>
|
||||||
|
<span class="n">api_result</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'https://mediawiki.org/wiki/HyperSwitch/errors/bad_request'</span>
|
||||||
|
<span class="ow">and</span> <span class="n">api_result</span><span class="p">[</span><span class="s1">'detail'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'title-invalid-characters'</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">_network</span><span class="o">.</span><span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">api_result</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">utils</span><span class="o">.</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'titles'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'display'</span><span class="p">)</span> <span class="ow">or</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'title'</span><span class="p">))</span>
|
||||||
|
<span class="n">wikipedia_link</span> <span class="o">=</span> <span class="n">api_result</span><span class="p">[</span><span class="s1">'content_urls'</span><span class="p">][</span><span class="s1">'desktop'</span><span class="p">][</span><span class="s1">'page'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s2">"list"</span> <span class="ow">in</span> <span class="n">display_type</span> <span class="ow">or</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'type'</span><span class="p">)</span> <span class="o">!=</span> <span class="s1">'standard'</span><span class="p">:</span>
|
||||||
|
<span class="c1"># show item in the result list if 'list' is in the display options or it</span>
|
||||||
|
<span class="c1"># is a item that can't be displayed in a infobox.</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">wikipedia_link</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'description'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)})</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s2">"infobox"</span> <span class="ow">in</span> <span class="n">display_type</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'type'</span><span class="p">)</span> <span class="o">==</span> <span class="s1">'standard'</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'infobox'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'id'</span><span class="p">:</span> <span class="n">wikipedia_link</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'extract'</span><span class="p">,</span> <span class="s1">''</span><span class="p">),</span>
|
||||||
|
<span class="s1">'img_src'</span><span class="p">:</span> <span class="n">api_result</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'thumbnail'</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'source'</span><span class="p">),</span>
|
||||||
|
<span class="s1">'urls'</span><span class="p">:</span> <span class="p">[{</span><span class="s1">'title'</span><span class="p">:</span> <span class="s1">'Wikipedia'</span><span class="p">,</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">wikipedia_link</span><span class="p">}],</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># Nonstandard language codes</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># These Wikipedias use language codes that do not conform to the ISO 639</span>
|
||||||
|
<span class="c1"># standard (which is how wiki subdomains are chosen nowadays).</span>
|
||||||
|
|
||||||
|
<span class="n">lang_map</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">LOCALE_BEST_MATCH</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||||
|
<span class="n">lang_map</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'be-tarask'</span><span class="p">:</span> <span class="s1">'bel'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ak'</span><span class="p">:</span> <span class="s1">'aka'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'als'</span><span class="p">:</span> <span class="s1">'gsw'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'bat-smg'</span><span class="p">:</span> <span class="s1">'sgs'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'cbk-zam'</span><span class="p">:</span> <span class="s1">'cbk'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'fiu-vro'</span><span class="p">:</span> <span class="s1">'vro'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'map-bms'</span><span class="p">:</span> <span class="s1">'map'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'no'</span><span class="p">:</span> <span class="s1">'nb-NO'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'nrm'</span><span class="p">:</span> <span class="s1">'nrf'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'roa-rup'</span><span class="p">:</span> <span class="s1">'rup'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'nds-nl'</span><span class="p">:</span> <span class="s1">'nds'</span><span class="p">,</span>
|
||||||
|
<span class="c1">#'simple: – invented code used for the Simple English Wikipedia (not the official IETF code en-simple)</span>
|
||||||
|
<span class="s1">'zh-min-nan'</span><span class="p">:</span> <span class="s1">'nan'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'zh-yue'</span><span class="p">:</span> <span class="s1">'yue'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'an'</span><span class="p">:</span> <span class="s1">'arg'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="n">fetch_wikimedia_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">)</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"WIKIPEDIA_LANGUAGES: </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="nb">len</span><span class="p">(</span><span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'WIKIPEDIA_LANGUAGES'</span><span class="p">]))</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_wikimedia_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/wikipedia.html#searx.engines.wikipedia.fetch_wikimedia_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_wikimedia_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages from Wikipedia. Not all languages from the</span>
|
||||||
|
<span class="sd"> :py:obj:`list_of_wikipedias` are supported by SearXNG locales, only those</span>
|
||||||
|
<span class="sd"> known from :py:obj:`searx.locales.LOCALE_NAMES` or those with a minimal</span>
|
||||||
|
<span class="sd"> :py:obj:`editing depth <wikipedia_article_depth>`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The location of the Wikipedia address of a language is mapped in a</span>
|
||||||
|
<span class="sd"> :py:obj:`custom field <searx.enginelib.traits.EngineTraits.custom>`</span>
|
||||||
|
<span class="sd"> (``wiki_netloc``). Here is a reduced example:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> traits.custom['wiki_netloc'] = {</span>
|
||||||
|
<span class="sd"> "en": "en.wikipedia.org",</span>
|
||||||
|
<span class="sd"> ..</span>
|
||||||
|
<span class="sd"> "gsw": "als.wikipedia.org",</span>
|
||||||
|
<span class="sd"> ..</span>
|
||||||
|
<span class="sd"> "zh": "zh.wikipedia.org",</span>
|
||||||
|
<span class="sd"> "zh-classical": "zh-classical.wikipedia.org"</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'WIKIPEDIA_LANGUAGES'</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="c1"># insert alias to map from a script or region to a wikipedia variant</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">sxng_tag_list</span> <span class="ow">in</span> <span class="n">wikipedia_script_variants</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">for</span> <span class="n">sxng_tag</span> <span class="ow">in</span> <span class="n">sxng_tag_list</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
<span class="k">for</span> <span class="n">eng_tag</span><span class="p">,</span> <span class="n">sxng_tag_list</span> <span class="ow">in</span> <span class="n">wiki_lc_locale_variants</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">for</span> <span class="n">sxng_tag</span> <span class="ow">in</span> <span class="n">sxng_tag_list</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">regions</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">_network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">list_of_wikipedias</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"ERROR: response from Wikipedia is not OK."</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//table[contains(@class,"sortable")]//tbody/tr'</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">cols</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'./td'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">cols</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">cols</span> <span class="o">=</span> <span class="p">[</span><span class="n">c</span><span class="o">.</span><span class="n">text_content</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cols</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">depth</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">cols</span><span class="p">[</span><span class="mi">11</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="s1">'0'</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">','</span><span class="p">,</span> <span class="s1">''</span><span class="p">))</span>
|
||||||
|
<span class="n">articles</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">cols</span><span class="p">[</span><span class="mi">4</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">','</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">','</span><span class="p">,</span> <span class="s1">''</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="n">eng_tag</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">3</span><span class="p">]</span>
|
||||||
|
<span class="n">wiki_url</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'./td[4]/a/@href'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">wiki_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">wiki_url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_tag</span> <span class="o">=</span> <span class="n">locales</span><span class="o">.</span><span class="n">language_tag</span><span class="p">(</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">),</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="c1"># print("ERROR: %s [%s] is unknown by babel" % (cols[0], eng_tag))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">finally</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'WIKIPEDIA_LANGUAGES'</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">eng_tag</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">sxng_tag</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">locales</span><span class="o">.</span><span class="n">LOCALE_NAMES</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">articles</span> <span class="o"><</span> <span class="mi">10000</span><span class="p">:</span>
|
||||||
|
<span class="c1"># exclude languages with too few articles</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">int</span><span class="p">(</span><span class="n">depth</span><span class="p">)</span> <span class="o"><</span> <span class="mi">20</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Rough indicator of a Wikipedia’s quality, showing how</span>
|
||||||
|
<span class="c1"># frequently its articles are updated.</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_tag</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_tag</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_tag</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_tag</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'wiki_netloc'</span><span class="p">][</span><span class="n">eng_tag</span><span class="p">]</span> <span class="o">=</span> <span class="n">wiki_url</span><span class="o">.</span><span class="n">netloc</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s1">'WIKIPEDIA_LANGUAGES'</span><span class="p">]</span><span class="o">.</span><span class="n">sort</span><span class="p">()</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
447
_modules/searx/engines/xpath.html
Normal file
@@ -0,0 +1,447 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.xpath — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.xpath</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.xpath</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""The XPath engine is a *generic* engine with which it is possible to configure</span>
|
||||||
|
<span class="sd">engines in the settings.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _XPath selector: https://quickref.me/xpath.html#xpath-selectors</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">Request:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`search_url`</span>
|
||||||
|
<span class="sd">- :py:obj:`lang_all`</span>
|
||||||
|
<span class="sd">- :py:obj:`soft_max_redirects`</span>
|
||||||
|
<span class="sd">- :py:obj:`method`</span>
|
||||||
|
<span class="sd">- :py:obj:`request_body`</span>
|
||||||
|
<span class="sd">- :py:obj:`cookies`</span>
|
||||||
|
<span class="sd">- :py:obj:`headers`</span>
|
||||||
|
|
||||||
|
<span class="sd">Paging:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`paging`</span>
|
||||||
|
<span class="sd">- :py:obj:`page_size`</span>
|
||||||
|
<span class="sd">- :py:obj:`first_page_num`</span>
|
||||||
|
|
||||||
|
<span class="sd">Time Range:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`time_range_support`</span>
|
||||||
|
<span class="sd">- :py:obj:`time_range_url`</span>
|
||||||
|
<span class="sd">- :py:obj:`time_range_map`</span>
|
||||||
|
|
||||||
|
<span class="sd">Safe-Search:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`safe_search_support`</span>
|
||||||
|
<span class="sd">- :py:obj:`safe_search_map`</span>
|
||||||
|
|
||||||
|
<span class="sd">Response:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`no_result_for_http_status`</span>
|
||||||
|
|
||||||
|
<span class="sd">`XPath selector`_:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`results_xpath`</span>
|
||||||
|
<span class="sd">- :py:obj:`url_xpath`</span>
|
||||||
|
<span class="sd">- :py:obj:`title_xpath`</span>
|
||||||
|
<span class="sd">- :py:obj:`content_xpath`</span>
|
||||||
|
<span class="sd">- :py:obj:`thumbnail_xpath`</span>
|
||||||
|
<span class="sd">- :py:obj:`suggestion_xpath`</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Example</span>
|
||||||
|
<span class="sd">=======</span>
|
||||||
|
|
||||||
|
<span class="sd">Here is a simple example of a XPath engine configured in the :ref:`settings</span>
|
||||||
|
<span class="sd">engines` section, further read :ref:`engines-dev`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name : bitbucket</span>
|
||||||
|
<span class="sd"> engine : xpath</span>
|
||||||
|
<span class="sd"> paging : True</span>
|
||||||
|
<span class="sd"> search_url : https://bitbucket.org/repo/all/{pageno}?name={query}</span>
|
||||||
|
<span class="sd"> url_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]/@href</span>
|
||||||
|
<span class="sd"> title_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]</span>
|
||||||
|
<span class="sd"> content_xpath : //article[@class="repo-summary"]/p</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlencode</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">extract_url</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_list</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">raise_for_httperror</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="n">search_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Search URL of the engine. Example::</span>
|
||||||
|
|
||||||
|
<span class="sd"> https://example.org/?search={query}&page={pageno}{time_range}{safe_search}</span>
|
||||||
|
|
||||||
|
<span class="sd">Replacements are:</span>
|
||||||
|
|
||||||
|
<span class="sd">``{query}``:</span>
|
||||||
|
<span class="sd"> Search terms from user.</span>
|
||||||
|
|
||||||
|
<span class="sd">``{pageno}``:</span>
|
||||||
|
<span class="sd"> Page number if engine supports paging :py:obj:`paging`</span>
|
||||||
|
|
||||||
|
<span class="sd">``{lang}``:</span>
|
||||||
|
<span class="sd"> ISO 639-1 language code (en, de, fr ..)</span>
|
||||||
|
|
||||||
|
<span class="sd">``{time_range}``:</span>
|
||||||
|
<span class="sd"> :py:obj:`URL parameter <time_range_url>` if engine :py:obj:`supports time</span>
|
||||||
|
<span class="sd"> range <time_range_support>`. The value for the parameter is taken from</span>
|
||||||
|
<span class="sd"> :py:obj:`time_range_map`.</span>
|
||||||
|
|
||||||
|
<span class="sd">``{safe_search}``:</span>
|
||||||
|
<span class="sd"> Safe-search :py:obj:`URL parameter <safe_search_map>` if engine</span>
|
||||||
|
<span class="sd"> :py:obj:`supports safe-search <safe_search_support>`. The ``{safe_search}``</span>
|
||||||
|
<span class="sd"> replacement is taken from the :py:obj:`safes_search_map`. Filter results::</span>
|
||||||
|
|
||||||
|
<span class="sd"> 0: none, 1: moderate, 2:strict</span>
|
||||||
|
|
||||||
|
<span class="sd"> If not supported, the URL parameter is an empty string.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">lang_all</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||||
|
<span class="sd">'''Replacement ``{lang}`` in :py:obj:`search_url` if language ``all`` is</span>
|
||||||
|
<span class="sd">selected.</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">no_result_for_http_status</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="sd">'''Return empty result for these HTTP status codes instead of throwing an error.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> no_result_for_http_status: []</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">soft_max_redirects</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="sd">'''Maximum redirects, soft limit. Record an error but don't stop the engine'''</span>
|
||||||
|
|
||||||
|
<span class="n">results_xpath</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="sd">'''`XPath selector`_ for the list of result items'''</span>
|
||||||
|
|
||||||
|
<span class="n">url_xpath</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">'''`XPath selector`_ of result's ``url``.'''</span>
|
||||||
|
|
||||||
|
<span class="n">content_xpath</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">'''`XPath selector`_ of result's ``content``.'''</span>
|
||||||
|
|
||||||
|
<span class="n">title_xpath</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="sd">'''`XPath selector`_ of result's ``title``.'''</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail_xpath</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''`XPath selector`_ of result's ``thumbnail``.'''</span>
|
||||||
|
|
||||||
|
<span class="n">suggestion_xpath</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="sd">'''`XPath selector`_ of result's ``suggestion``.'''</span>
|
||||||
|
|
||||||
|
<span class="n">cached_xpath</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="n">cached_url</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
|
||||||
|
<span class="n">cookies</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="sd">'''Some engines might offer different result based on cookies.</span>
|
||||||
|
<span class="sd">Possible use-case: To set safesearch cookie.'''</span>
|
||||||
|
|
||||||
|
<span class="n">headers</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="sd">'''Some engines might offer different result based headers. Possible use-case:</span>
|
||||||
|
<span class="sd">To set header to moderate.'''</span>
|
||||||
|
|
||||||
|
<span class="n">method</span> <span class="o">=</span> <span class="s1">'GET'</span>
|
||||||
|
<span class="sd">'''Some engines might require to do POST requests for search.'''</span>
|
||||||
|
|
||||||
|
<span class="n">request_body</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="sd">'''The body of the request. This can only be used if different :py:obj:`method`</span>
|
||||||
|
<span class="sd">is set, e.g. ``POST``. For formatting see the documentation of :py:obj:`search_url`::</span>
|
||||||
|
|
||||||
|
<span class="sd"> search={query}&page={pageno}{time_range}{safe_search}</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''Engine supports paging [True or False].'''</span>
|
||||||
|
|
||||||
|
<span class="n">page_size</span> <span class="o">=</span> <span class="mi">1</span>
|
||||||
|
<span class="sd">'''Number of results on each page. Only needed if the site requires not a page</span>
|
||||||
|
<span class="sd">number, but an offset.'''</span>
|
||||||
|
|
||||||
|
<span class="n">first_page_num</span> <span class="o">=</span> <span class="mi">1</span>
|
||||||
|
<span class="sd">'''Number of the first page (usually 0 or 1).'''</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''Engine supports search time range.'''</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_url</span> <span class="o">=</span> <span class="s1">'&hours=</span><span class="si">{time_range_val}</span><span class="s1">'</span>
|
||||||
|
<span class="sd">'''Time range URL parameter in the in :py:obj:`search_url`. If no time range is</span>
|
||||||
|
<span class="sd">requested by the user, the URL parameter is an empty string. The</span>
|
||||||
|
<span class="sd">``{time_range_val}`` replacement is taken from the :py:obj:`time_range_map`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> time_range_url : '&days={time_range_val}'</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_map</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'day'</span><span class="p">:</span> <span class="mi">24</span><span class="p">,</span>
|
||||||
|
<span class="s1">'week'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
|
||||||
|
<span class="s1">'month'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">30</span><span class="p">,</span>
|
||||||
|
<span class="s1">'year'</span><span class="p">:</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">365</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">'''Maps time range value from user to ``{time_range_val}`` in</span>
|
||||||
|
<span class="sd">:py:obj:`time_range_url`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> time_range_map:</span>
|
||||||
|
<span class="sd"> day: 1</span>
|
||||||
|
<span class="sd"> week: 7</span>
|
||||||
|
<span class="sd"> month: 30</span>
|
||||||
|
<span class="sd"> year: 365</span>
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
<span class="n">safe_search_support</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="sd">'''Engine supports safe-search.'''</span>
|
||||||
|
|
||||||
|
<span class="n">safe_search_map</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'&filter=none'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'&filter=moderate'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'&filter=strict'</span><span class="p">}</span>
|
||||||
|
<span class="sd">'''Maps safe-search value to ``{safe_search}`` in :py:obj:`search_url`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> safesearch: true</span>
|
||||||
|
<span class="sd"> safes_search_map:</span>
|
||||||
|
<span class="sd"> 0: '&filter=none'</span>
|
||||||
|
<span class="sd"> 1: '&filter=moderate'</span>
|
||||||
|
<span class="sd"> 2: '&filter=strict'</span>
|
||||||
|
|
||||||
|
<span class="sd">'''</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/xpath.html#searx.engines.xpath.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">'''Build request parameters (see :ref:`engine request`).'''</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">lang_all</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'all'</span><span class="p">:</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'language'</span><span class="p">][:</span><span class="mi">2</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">time_range</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'time_range'</span><span class="p">):</span>
|
||||||
|
<span class="n">time_range_val</span> <span class="o">=</span> <span class="n">time_range_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'time_range'</span><span class="p">))</span>
|
||||||
|
<span class="n">time_range</span> <span class="o">=</span> <span class="n">time_range_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">time_range_val</span><span class="o">=</span><span class="n">time_range_val</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">safe_search</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]:</span>
|
||||||
|
<span class="n">safe_search</span> <span class="o">=</span> <span class="n">safe_search_map</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="n">fargs</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'query'</span><span class="p">:</span> <span class="n">urlencode</span><span class="p">({</span><span class="s1">'q'</span><span class="p">:</span> <span class="n">query</span><span class="p">})[</span><span class="mi">2</span><span class="p">:],</span>
|
||||||
|
<span class="s1">'lang'</span><span class="p">:</span> <span class="n">lang</span><span class="p">,</span>
|
||||||
|
<span class="s1">'pageno'</span><span class="p">:</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">page_size</span> <span class="o">+</span> <span class="n">first_page_num</span><span class="p">,</span>
|
||||||
|
<span class="s1">'time_range'</span><span class="p">:</span> <span class="n">time_range</span><span class="p">,</span>
|
||||||
|
<span class="s1">'safe_search'</span><span class="p">:</span> <span class="n">safe_search</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">cookies</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'headers'</span><span class="p">]</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">headers</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fargs</span><span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'method'</span><span class="p">]</span> <span class="o">=</span> <span class="n">method</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">request_body</span><span class="p">:</span>
|
||||||
|
<span class="c1"># don't url-encode the query if it's in the request body</span>
|
||||||
|
<span class="n">fargs</span><span class="p">[</span><span class="s1">'query'</span><span class="p">]</span> <span class="o">=</span> <span class="n">query</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'data'</span><span class="p">]</span> <span class="o">=</span> <span class="n">request_body</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">fargs</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'soft_max_redirects'</span><span class="p">]</span> <span class="o">=</span> <span class="n">soft_max_redirects</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'raise_for_httperror'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">params</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/xpath.html#searx.engines.xpath.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span> <span class="c1"># pylint: disable=too-many-branches</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Scrap *results* from the response (see :ref:`result types`)."""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">no_result_for_http_status</span> <span class="ow">and</span> <span class="n">resp</span><span class="o">.</span><span class="n">status_code</span> <span class="ow">in</span> <span class="n">no_result_for_http_status</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">raise_for_httperror</span><span class="p">(</span><span class="n">resp</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
<span class="n">is_onion</span> <span class="o">=</span> <span class="s1">'onions'</span> <span class="ow">in</span> <span class="n">categories</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">results_xpath</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">results_xpath</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">extract_url</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span> <span class="n">search_url</span><span class="p">)</span>
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">))</span>
|
||||||
|
<span class="n">tmp_result</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># add thumbnail if available</span>
|
||||||
|
<span class="k">if</span> <span class="n">thumbnail_xpath</span><span class="p">:</span>
|
||||||
|
<span class="n">thumbnail_xpath_result</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">thumbnail_xpath</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">thumbnail_xpath_result</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'thumbnail'</span><span class="p">]</span> <span class="o">=</span> <span class="n">extract_url</span><span class="p">(</span><span class="n">thumbnail_xpath_result</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># add alternative cached url if available</span>
|
||||||
|
<span class="k">if</span> <span class="n">cached_xpath</span><span class="p">:</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'cached_url'</span><span class="p">]</span> <span class="o">=</span> <span class="n">cached_url</span> <span class="o">+</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">cached_xpath</span><span class="p">,</span> <span class="n">min_len</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">is_onion</span><span class="p">:</span>
|
||||||
|
<span class="n">tmp_result</span><span class="p">[</span><span class="s1">'is_onion'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tmp_result</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">cached_xpath</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">url</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">cached</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span>
|
||||||
|
<span class="p">(</span><span class="n">extract_url</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">)),</span>
|
||||||
|
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">)),</span>
|
||||||
|
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">)),</span>
|
||||||
|
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">cached_xpath</span><span class="p">)),</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span>
|
||||||
|
<span class="s1">'cached_url'</span><span class="p">:</span> <span class="n">cached_url</span> <span class="o">+</span> <span class="n">cached</span><span class="p">,</span>
|
||||||
|
<span class="s1">'is_onion'</span><span class="p">:</span> <span class="n">is_onion</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">url</span><span class="p">,</span> <span class="n">title</span><span class="p">,</span> <span class="n">content</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span>
|
||||||
|
<span class="p">(</span><span class="n">extract_url</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">search_url</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">)),</span>
|
||||||
|
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">)),</span>
|
||||||
|
<span class="nb">map</span><span class="p">(</span><span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">content_xpath</span><span class="p">)),</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span> <span class="s1">'title'</span><span class="p">:</span> <span class="n">title</span><span class="p">,</span> <span class="s1">'content'</span><span class="p">:</span> <span class="n">content</span><span class="p">,</span> <span class="s1">'is_onion'</span><span class="p">:</span> <span class="n">is_onion</span><span class="p">})</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">suggestion_xpath</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="n">suggestion_xpath</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"found </span><span class="si">%s</span><span class="s2"> results"</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">results</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
377
_modules/searx/engines/yahoo.html
Normal file
@@ -0,0 +1,377 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.yahoo — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.yahoo</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.yahoo</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Yahoo Search (Web)</span>
|
||||||
|
|
||||||
|
<span class="sd">Languages are supported by mapping the language to a domain. If domain is not</span>
|
||||||
|
<span class="sd">found in :py:obj:`lang2domain` URL ``<lang>.search.yahoo.com`` is used.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">unquote</span><span class="p">,</span>
|
||||||
|
<span class="n">urlencode</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">eval_xpath_getindex</span><span class="p">,</span>
|
||||||
|
<span class="n">eval_xpath_list</span><span class="p">,</span>
|
||||||
|
<span class="n">extract_text</span><span class="p">,</span>
|
||||||
|
<span class="n">html_to_text</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># about</span>
|
||||||
|
<span class="n">about</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s1">'https://search.yahoo.com/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="s1">'https://developer.yahoo.com/api/'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s1">'HTML'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># engine dependent config</span>
|
||||||
|
<span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'general'</span><span class="p">,</span> <span class="s1">'web'</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">time_range_support</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="c1"># send_accept_language_header = True</span>
|
||||||
|
|
||||||
|
<span class="n">time_range_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'day'</span><span class="p">:</span> <span class="s1">'d'</span><span class="p">,</span> <span class="s1">'week'</span><span class="p">:</span> <span class="s1">'w'</span><span class="p">,</span> <span class="s1">'month'</span><span class="p">:</span> <span class="s1">'m'</span><span class="p">}</span>
|
||||||
|
<span class="n">safesearch_dict</span> <span class="o">=</span> <span class="p">{</span><span class="mi">0</span><span class="p">:</span> <span class="s1">'p'</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span> <span class="s1">'i'</span><span class="p">,</span> <span class="mi">2</span><span class="p">:</span> <span class="s1">'r'</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">region2domain</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"CO"</span><span class="p">:</span> <span class="s2">"co.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Colombia</span>
|
||||||
|
<span class="s2">"TH"</span><span class="p">:</span> <span class="s2">"th.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Thailand</span>
|
||||||
|
<span class="s2">"VE"</span><span class="p">:</span> <span class="s2">"ve.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Venezuela</span>
|
||||||
|
<span class="s2">"CL"</span><span class="p">:</span> <span class="s2">"cl.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Chile</span>
|
||||||
|
<span class="s2">"HK"</span><span class="p">:</span> <span class="s2">"hk.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Hong Kong</span>
|
||||||
|
<span class="s2">"PE"</span><span class="p">:</span> <span class="s2">"pe.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Peru</span>
|
||||||
|
<span class="s2">"CA"</span><span class="p">:</span> <span class="s2">"ca.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Canada</span>
|
||||||
|
<span class="s2">"DE"</span><span class="p">:</span> <span class="s2">"de.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Germany</span>
|
||||||
|
<span class="s2">"FR"</span><span class="p">:</span> <span class="s2">"fr.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># France</span>
|
||||||
|
<span class="s2">"TW"</span><span class="p">:</span> <span class="s2">"tw.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Taiwan</span>
|
||||||
|
<span class="s2">"GB"</span><span class="p">:</span> <span class="s2">"uk.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># United Kingdom</span>
|
||||||
|
<span class="s2">"UK"</span><span class="p">:</span> <span class="s2">"uk.search.yahoo.com"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"BR"</span><span class="p">:</span> <span class="s2">"br.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Brazil</span>
|
||||||
|
<span class="s2">"IN"</span><span class="p">:</span> <span class="s2">"in.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># India</span>
|
||||||
|
<span class="s2">"ES"</span><span class="p">:</span> <span class="s2">"espanol.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Espanol</span>
|
||||||
|
<span class="s2">"PH"</span><span class="p">:</span> <span class="s2">"ph.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Philippines</span>
|
||||||
|
<span class="s2">"AR"</span><span class="p">:</span> <span class="s2">"ar.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Argentina</span>
|
||||||
|
<span class="s2">"MX"</span><span class="p">:</span> <span class="s2">"mx.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Mexico</span>
|
||||||
|
<span class="s2">"SG"</span><span class="p">:</span> <span class="s2">"sg.search.yahoo.com"</span><span class="p">,</span> <span class="c1"># Singapore</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""Map regions to domain"""</span>
|
||||||
|
|
||||||
|
<span class="n">lang2domain</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'zh_chs'</span><span class="p">:</span> <span class="s1">'hk.search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'zh_cht'</span><span class="p">:</span> <span class="s1">'tw.search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'any'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'en'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'bg'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'cs'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'da'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'el'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'et'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'he'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'hr'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ja'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'ko'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'sk'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'sl'</span><span class="p">:</span> <span class="s1">'search.yahoo.com'</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""Map language to domain"""</span>
|
||||||
|
|
||||||
|
<span class="n">yahoo_languages</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"all"</span><span class="p">:</span> <span class="s2">"any"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"ar"</span><span class="p">:</span> <span class="s2">"ar"</span><span class="p">,</span> <span class="c1"># Arabic</span>
|
||||||
|
<span class="s2">"bg"</span><span class="p">:</span> <span class="s2">"bg"</span><span class="p">,</span> <span class="c1"># Bulgarian</span>
|
||||||
|
<span class="s2">"cs"</span><span class="p">:</span> <span class="s2">"cs"</span><span class="p">,</span> <span class="c1"># Czech</span>
|
||||||
|
<span class="s2">"da"</span><span class="p">:</span> <span class="s2">"da"</span><span class="p">,</span> <span class="c1"># Danish</span>
|
||||||
|
<span class="s2">"de"</span><span class="p">:</span> <span class="s2">"de"</span><span class="p">,</span> <span class="c1"># German</span>
|
||||||
|
<span class="s2">"el"</span><span class="p">:</span> <span class="s2">"el"</span><span class="p">,</span> <span class="c1"># Greek</span>
|
||||||
|
<span class="s2">"en"</span><span class="p">:</span> <span class="s2">"en"</span><span class="p">,</span> <span class="c1"># English</span>
|
||||||
|
<span class="s2">"es"</span><span class="p">:</span> <span class="s2">"es"</span><span class="p">,</span> <span class="c1"># Spanish</span>
|
||||||
|
<span class="s2">"et"</span><span class="p">:</span> <span class="s2">"et"</span><span class="p">,</span> <span class="c1"># Estonian</span>
|
||||||
|
<span class="s2">"fi"</span><span class="p">:</span> <span class="s2">"fi"</span><span class="p">,</span> <span class="c1"># Finnish</span>
|
||||||
|
<span class="s2">"fr"</span><span class="p">:</span> <span class="s2">"fr"</span><span class="p">,</span> <span class="c1"># French</span>
|
||||||
|
<span class="s2">"he"</span><span class="p">:</span> <span class="s2">"he"</span><span class="p">,</span> <span class="c1"># Hebrew</span>
|
||||||
|
<span class="s2">"hr"</span><span class="p">:</span> <span class="s2">"hr"</span><span class="p">,</span> <span class="c1"># Croatian</span>
|
||||||
|
<span class="s2">"hu"</span><span class="p">:</span> <span class="s2">"hu"</span><span class="p">,</span> <span class="c1"># Hungarian</span>
|
||||||
|
<span class="s2">"it"</span><span class="p">:</span> <span class="s2">"it"</span><span class="p">,</span> <span class="c1"># Italian</span>
|
||||||
|
<span class="s2">"ja"</span><span class="p">:</span> <span class="s2">"ja"</span><span class="p">,</span> <span class="c1"># Japanese</span>
|
||||||
|
<span class="s2">"ko"</span><span class="p">:</span> <span class="s2">"ko"</span><span class="p">,</span> <span class="c1"># Korean</span>
|
||||||
|
<span class="s2">"lt"</span><span class="p">:</span> <span class="s2">"lt"</span><span class="p">,</span> <span class="c1"># Lithuanian</span>
|
||||||
|
<span class="s2">"lv"</span><span class="p">:</span> <span class="s2">"lv"</span><span class="p">,</span> <span class="c1"># Latvian</span>
|
||||||
|
<span class="s2">"nl"</span><span class="p">:</span> <span class="s2">"nl"</span><span class="p">,</span> <span class="c1"># Dutch</span>
|
||||||
|
<span class="s2">"no"</span><span class="p">:</span> <span class="s2">"no"</span><span class="p">,</span> <span class="c1"># Norwegian</span>
|
||||||
|
<span class="s2">"pl"</span><span class="p">:</span> <span class="s2">"pl"</span><span class="p">,</span> <span class="c1"># Polish</span>
|
||||||
|
<span class="s2">"pt"</span><span class="p">:</span> <span class="s2">"pt"</span><span class="p">,</span> <span class="c1"># Portuguese</span>
|
||||||
|
<span class="s2">"ro"</span><span class="p">:</span> <span class="s2">"ro"</span><span class="p">,</span> <span class="c1"># Romanian</span>
|
||||||
|
<span class="s2">"ru"</span><span class="p">:</span> <span class="s2">"ru"</span><span class="p">,</span> <span class="c1"># Russian</span>
|
||||||
|
<span class="s2">"sk"</span><span class="p">:</span> <span class="s2">"sk"</span><span class="p">,</span> <span class="c1"># Slovak</span>
|
||||||
|
<span class="s2">"sl"</span><span class="p">:</span> <span class="s2">"sl"</span><span class="p">,</span> <span class="c1"># Slovenian</span>
|
||||||
|
<span class="s2">"sv"</span><span class="p">:</span> <span class="s2">"sv"</span><span class="p">,</span> <span class="c1"># Swedish</span>
|
||||||
|
<span class="s2">"th"</span><span class="p">:</span> <span class="s2">"th"</span><span class="p">,</span> <span class="c1"># Thai</span>
|
||||||
|
<span class="s2">"tr"</span><span class="p">:</span> <span class="s2">"tr"</span><span class="p">,</span> <span class="c1"># Turkish</span>
|
||||||
|
<span class="s2">"zh"</span><span class="p">:</span> <span class="s2">"zh_chs"</span><span class="p">,</span> <span class="c1"># Chinese (Simplified)</span>
|
||||||
|
<span class="s2">"zh_Hans"</span><span class="p">:</span> <span class="s2">"zh_chs"</span><span class="p">,</span>
|
||||||
|
<span class="s1">'zh-CN'</span><span class="p">:</span> <span class="s2">"zh_chs"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"zh_Hant"</span><span class="p">:</span> <span class="s2">"zh_cht"</span><span class="p">,</span> <span class="c1"># Chinese (Traditional)</span>
|
||||||
|
<span class="s2">"zh-HK"</span><span class="p">:</span> <span class="s2">"zh_cht"</span><span class="p">,</span>
|
||||||
|
<span class="s1">'zh-TW'</span><span class="p">:</span> <span class="s2">"zh_cht"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="build_sb_cookie">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/yahoo.html#searx.engines.yahoo.build_sb_cookie">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">build_sb_cookie</span><span class="p">(</span><span class="n">cookie_params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Build sB cookie parameter from provided parameters.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param cookie_params: Dictionary of cookie parameters</span>
|
||||||
|
<span class="sd"> :type cookie_params: dict</span>
|
||||||
|
<span class="sd"> :returns: Formatted cookie string</span>
|
||||||
|
<span class="sd"> :rtype: str</span>
|
||||||
|
|
||||||
|
<span class="sd"> Example:</span>
|
||||||
|
<span class="sd"> >>> cookie_params = {'v': '1', 'vm': 'p', 'fl': '1', 'vl': 'lang_fr'}</span>
|
||||||
|
<span class="sd"> >>> build_sb_cookie(cookie_params)</span>
|
||||||
|
<span class="sd"> 'v=1&vm=p&fl=1&vl=lang_fr'</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">cookie_parts</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">cookie_params</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">cookie_parts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2">=</span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="s2">"&"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">cookie_parts</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="request">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/yahoo.html#searx.engines.yahoo.request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Build Yahoo search request."""</span>
|
||||||
|
|
||||||
|
<span class="n">lang</span><span class="p">,</span> <span class="n">region</span> <span class="o">=</span> <span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"language"</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"-"</span><span class="p">)</span> <span class="o">+</span> <span class="p">[</span><span class="kc">None</span><span class="p">])[:</span><span class="mi">2</span><span class="p">]</span>
|
||||||
|
<span class="n">lang</span> <span class="o">=</span> <span class="n">yahoo_languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="s2">"any"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Build URL parameters</span>
|
||||||
|
<span class="c1"># - p (str): Search query string</span>
|
||||||
|
<span class="c1"># - btf (str): Time filter, maps to values like 'd' (day), 'w' (week), 'm' (month)</span>
|
||||||
|
<span class="c1"># - iscqry (str): Empty string, necessary for results to appear properly on first page</span>
|
||||||
|
<span class="c1"># - b (int): Search offset for pagination</span>
|
||||||
|
<span class="c1"># - pz (str): Amount of results expected for the page</span>
|
||||||
|
<span class="n">url_params</span> <span class="o">=</span> <span class="p">{</span><span class="s1">'p'</span><span class="p">:</span> <span class="n">query</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">btf</span> <span class="o">=</span> <span class="n">time_range_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s1">'time_range'</span><span class="p">])</span>
|
||||||
|
<span class="k">if</span> <span class="n">btf</span><span class="p">:</span>
|
||||||
|
<span class="n">url_params</span><span class="p">[</span><span class="s1">'btf'</span><span class="p">]</span> <span class="o">=</span> <span class="n">btf</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">url_params</span><span class="p">[</span><span class="s1">'iscqry'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">''</span>
|
||||||
|
<span class="k">elif</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">>=</span> <span class="mi">2</span><span class="p">:</span>
|
||||||
|
<span class="n">url_params</span><span class="p">[</span><span class="s1">'b'</span><span class="p">]</span> <span class="o">=</span> <span class="n">params</span><span class="p">[</span><span class="s1">'pageno'</span><span class="p">]</span> <span class="o">*</span> <span class="mi">7</span> <span class="o">+</span> <span class="mi">1</span> <span class="c1"># 8, 15, 21, etc.</span>
|
||||||
|
<span class="n">url_params</span><span class="p">[</span><span class="s1">'pz'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">7</span>
|
||||||
|
<span class="n">url_params</span><span class="p">[</span><span class="s1">'bct'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="n">url_params</span><span class="p">[</span><span class="s1">'xargs'</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
|
||||||
|
<span class="c1"># Build sB cookie (for filters)</span>
|
||||||
|
<span class="c1"># - vm (str): SafeSearch filter, maps to values like 'p' (None), 'i' (Moderate), 'r' (Strict)</span>
|
||||||
|
<span class="c1"># - fl (bool): Indicates if a search language is used or not</span>
|
||||||
|
<span class="c1"># - vl (str): The search language to use (e.g. lang_fr)</span>
|
||||||
|
<span class="n">sbcookie_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s1">'v'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="s1">'vm'</span><span class="p">:</span> <span class="n">safesearch_dict</span><span class="p">[</span><span class="n">params</span><span class="p">[</span><span class="s1">'safesearch'</span><span class="p">]],</span>
|
||||||
|
<span class="s1">'fl'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="s1">'vl'</span><span class="p">:</span> <span class="sa">f</span><span class="s1">'lang_</span><span class="si">{</span><span class="n">lang</span><span class="si">}</span><span class="s1">'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'pn'</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
|
||||||
|
<span class="s1">'rw'</span><span class="p">:</span> <span class="s1">'new'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'userset'</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'cookies'</span><span class="p">][</span><span class="s1">'sB'</span><span class="p">]</span> <span class="o">=</span> <span class="n">build_sb_cookie</span><span class="p">(</span><span class="n">sbcookie_params</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Search region/language</span>
|
||||||
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">region2domain</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">region</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">domain</span><span class="p">:</span>
|
||||||
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">lang2domain</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">lang</span><span class="p">,</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">lang</span><span class="si">}</span><span class="s1">.search.yahoo.com'</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s1">'domain selected: </span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s1">'cookies: </span><span class="si">{</span><span class="n">params</span><span class="p">[</span><span class="s2">"cookies"</span><span class="p">]</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'url'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'https://</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s1">/search?</span><span class="si">{</span><span class="n">urlencode</span><span class="p">(</span><span class="n">url_params</span><span class="p">)</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s1">'domain'</span><span class="p">]</span> <span class="o">=</span> <span class="n">domain</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="parse_url">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/yahoo.html#searx.engines.yahoo.parse_url">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">parse_url</span><span class="p">(</span><span class="n">url_string</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""remove yahoo-specific tracking-url"""</span>
|
||||||
|
|
||||||
|
<span class="n">endings</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'/RS'</span><span class="p">,</span> <span class="s1">'/RK'</span><span class="p">]</span>
|
||||||
|
<span class="n">endpositions</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">start</span> <span class="o">=</span> <span class="n">url_string</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'http'</span><span class="p">,</span> <span class="n">url_string</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'/RU='</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">ending</span> <span class="ow">in</span> <span class="n">endings</span><span class="p">:</span>
|
||||||
|
<span class="n">endpos</span> <span class="o">=</span> <span class="n">url_string</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="n">ending</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">endpos</span> <span class="o">></span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="n">endpositions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">endpos</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">start</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">endpositions</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">url_string</span>
|
||||||
|
|
||||||
|
<span class="n">end</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">endpositions</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">unquote</span><span class="p">(</span><span class="n">url_string</span><span class="p">[</span><span class="n">start</span><span class="p">:</span><span class="n">end</span><span class="p">])</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="response">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/yahoo.html#searx.engines.yahoo.response">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""parse response"""</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">url_xpath</span> <span class="o">=</span> <span class="s1">'.//div[contains(@class,"compTitle")]/h3/a/@href'</span>
|
||||||
|
<span class="n">title_xpath</span> <span class="o">=</span> <span class="s1">'.//h3//a/@aria-label'</span>
|
||||||
|
|
||||||
|
<span class="n">domain</span> <span class="o">=</span> <span class="n">resp</span><span class="o">.</span><span class="n">search_params</span><span class="p">[</span><span class="s1">'domain'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">domain</span> <span class="o">==</span> <span class="s2">"search.yahoo.com"</span><span class="p">:</span>
|
||||||
|
<span class="n">url_xpath</span> <span class="o">=</span> <span class="s1">'.//div[contains(@class,"compTitle")]/a/@href'</span>
|
||||||
|
<span class="n">title_xpath</span> <span class="o">=</span> <span class="s1">'.//div[contains(@class,"compTitle")]/a/h3/span'</span>
|
||||||
|
|
||||||
|
<span class="c1"># parse results</span>
|
||||||
|
<span class="k">for</span> <span class="n">result</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class,"algo-sr")]'</span><span class="p">):</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">url_xpath</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">parse_url</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">title</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">title_xpath</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span> <span class="o">=</span> <span class="n">eval_xpath_getindex</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "compText")]'</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="n">allow_none</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># append result</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="p">{</span>
|
||||||
|
<span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">,</span>
|
||||||
|
<span class="c1"># title sometimes contains HTML tags / see</span>
|
||||||
|
<span class="c1"># https://github.com/searxng/searxng/issues/3790</span>
|
||||||
|
<span class="s1">'title'</span><span class="p">:</span> <span class="s2">" "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">title</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">()),</span>
|
||||||
|
<span class="s1">'content'</span><span class="p">:</span> <span class="s2">" "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">html_to_text</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">()),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">suggestion</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s1">'//div[contains(@class, "AlsoTry")]//table//a'</span><span class="p">):</span>
|
||||||
|
<span class="c1"># append suggestion</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">({</span><span class="s1">'suggestion'</span><span class="p">:</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">suggestion</span><span class="p">)})</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
370
_modules/searx/engines/zlibrary.html
Normal file
@@ -0,0 +1,370 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.engines.zlibrary — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../engines.html" accesskey="U">searx.engines</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.engines.zlibrary</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.engines.zlibrary</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""`Z-Library`_ (abbreviated as z-lib, formerly BookFinder) is a shadow library</span>
|
||||||
|
<span class="sd">project for file-sharing access to scholarly journal articles, academic texts</span>
|
||||||
|
<span class="sd">and general-interest books. It began as a mirror of Library Genesis, from which</span>
|
||||||
|
<span class="sd">most of its books originate.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Z-Library: https://zlibrary-global.se/</span>
|
||||||
|
|
||||||
|
<span class="sd">Configuration</span>
|
||||||
|
<span class="sd">=============</span>
|
||||||
|
|
||||||
|
<span class="sd">The engine has the following additional settings:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`zlib_year_from`</span>
|
||||||
|
<span class="sd">- :py:obj:`zlib_year_to`</span>
|
||||||
|
<span class="sd">- :py:obj:`zlib_ext`</span>
|
||||||
|
|
||||||
|
<span class="sd">With this options a SearXNG maintainer is able to configure **additional**</span>
|
||||||
|
<span class="sd">engines for specific searches in Z-Library. For example a engine to search</span>
|
||||||
|
<span class="sd">only for EPUB from 2010 to 2020.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> - name: z-library 2010s epub</span>
|
||||||
|
<span class="sd"> engine: zlibrary</span>
|
||||||
|
<span class="sd"> shortcut: zlib2010s</span>
|
||||||
|
<span class="sd"> zlib_year_from: '2010'</span>
|
||||||
|
<span class="sd"> zlib_year_to: '2020'</span>
|
||||||
|
<span class="sd"> zlib_ext: 'EPUB'</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementations</span>
|
||||||
|
<span class="sd">===============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">quote</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">html</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">extract_text</span><span class="p">,</span> <span class="n">eval_xpath</span><span class="p">,</span> <span class="n">eval_xpath_list</span><span class="p">,</span> <span class="n">ElementType</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.enginelib.traits</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineTraits</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">ENGINE_TRAITS</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Response</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParams</span>
|
||||||
|
|
||||||
|
<span class="n">about</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"website"</span><span class="p">:</span> <span class="s2">"https://zlibrary-global.se"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"wikidata_id"</span><span class="p">:</span> <span class="s2">"Q104863992"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"official_api_documentation"</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
|
<span class="s2">"use_official_api"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"require_api_key"</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
|
<span class="s2">"results"</span><span class="p">:</span> <span class="s2">"HTML"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">categories</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"files"</span><span class="p">,</span> <span class="s2">"books"</span><span class="p">]</span>
|
||||||
|
<span class="n">paging</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="n">base_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"https://zlibrary-global.se"</span>
|
||||||
|
|
||||||
|
<span class="n">zlib_year_from</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Filter z-library's results by year from. E.g '2010'.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">zlib_year_to</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Filter z-library's results by year to. E.g. '2010'.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">zlib_ext</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="sd">"""Filter z-library's results by a file ending. Common filters for example are</span>
|
||||||
|
<span class="sd">``PDF`` and ``EPUB``.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">i18n_language</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"Language"</span><span class="p">)</span>
|
||||||
|
<span class="n">i18n_book_rating</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"Book rating"</span><span class="p">)</span>
|
||||||
|
<span class="n">i18n_file_quality</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"File quality"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="setup">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/zlibrary.html#searx.engines.zlibrary.setup">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">setup</span><span class="p">(</span><span class="n">engine_settings</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Check of engine's settings."""</span>
|
||||||
|
<span class="n">traits</span><span class="p">:</span> <span class="n">EngineTraits</span> <span class="o">=</span> <span class="n">EngineTraits</span><span class="p">(</span><span class="o">**</span><span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">"z-library"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">zlib_ext</span> <span class="ow">and</span> <span class="n">zlib_ext</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ext"</span><span class="p">]:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"invalid setting ext: </span><span class="si">{</span><span class="n">zlib_ext</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">zlib_year_from</span> <span class="ow">and</span> <span class="n">zlib_year_from</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"year_from"</span><span class="p">]:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"invalid setting year_from: </span><span class="si">{</span><span class="n">zlib_year_from</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">zlib_year_to</span> <span class="ow">and</span> <span class="n">zlib_year_to</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"year_to"</span><span class="p">]:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"invalid setting year_to: </span><span class="si">{</span><span class="n">zlib_year_to</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">request</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">params</span><span class="p">:</span> <span class="s2">"OnlineParams"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">lang</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="n">traits</span><span class="o">.</span><span class="n">get_language</span><span class="p">(</span><span class="n">params</span><span class="p">[</span><span class="s2">"searxng_locale"</span><span class="p">],</span> <span class="n">traits</span><span class="o">.</span><span class="n">all_locale</span><span class="p">)</span>
|
||||||
|
<span class="n">search_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="n">base_url</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"/s/</span><span class="si">{search_query}</span><span class="s2">/?page=</span><span class="si">{pageno}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"&yearFrom=</span><span class="si">{zlib_year_from}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"&yearTo=</span><span class="si">{zlib_year_to}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"&languages[]=</span><span class="si">{lang}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="s2">"&extensions[]=</span><span class="si">{zlib_ext}</span><span class="s2">"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="n">search_url</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||||
|
<span class="n">search_query</span><span class="o">=</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">),</span>
|
||||||
|
<span class="n">pageno</span><span class="o">=</span><span class="n">params</span><span class="p">[</span><span class="s2">"pageno"</span><span class="p">],</span>
|
||||||
|
<span class="n">lang</span><span class="o">=</span><span class="n">lang</span><span class="p">,</span>
|
||||||
|
<span class="n">zlib_year_from</span><span class="o">=</span><span class="n">zlib_year_from</span><span class="p">,</span>
|
||||||
|
<span class="n">zlib_year_to</span><span class="o">=</span><span class="n">zlib_year_to</span><span class="p">,</span>
|
||||||
|
<span class="n">zlib_ext</span><span class="o">=</span><span class="n">zlib_ext</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">params</span><span class="p">[</span><span class="s2">"verify"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">response</span><span class="p">(</span><span class="n">resp</span><span class="p">:</span> <span class="s2">"SXNG_Response"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">domain_is_seized</span><span class="p">(</span><span class="n">dom</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="n">SearxException</span><span class="p">(</span><span class="sa">f</span><span class="s2">"zlibrary domain is seized: </span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//div[@id="searchResultBox"]//div[contains(@class, "resItemBox")]'</span><span class="p">):</span>
|
||||||
|
<span class="n">kwargs</span> <span class="o">=</span> <span class="n">_parse_result</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="n">res</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">res</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Paper</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">res</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">domain_is_seized</span><span class="p">(</span><span class="n">dom</span><span class="p">:</span> <span class="n">ElementType</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//title'</span><span class="p">)</span> <span class="ow">and</span> <span class="s2">"seized"</span> <span class="ow">in</span> <span class="n">dom</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'//title'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">text</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_text</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">ElementType</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">extract_text</span><span class="p">(</span><span class="n">eval_xpath</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">selector</span><span class="p">))</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_result</span><span class="p">(</span><span class="n">item</span><span class="p">:</span> <span class="n">ElementType</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="n">author_elements</span> <span class="o">=</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//div[@class="authors"]//a[@itemprop="author"]'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">result</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"url"</span><span class="p">:</span> <span class="n">base_url</span> <span class="o">+</span> <span class="n">item</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s1">'(.//a[starts-with(@href, "/book/")])[1]/@href'</span><span class="p">)[</span><span class="mi">0</span><span class="p">],</span>
|
||||||
|
<span class="s2">"title"</span><span class="p">:</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//*[@itemprop="name"]'</span><span class="p">),</span>
|
||||||
|
<span class="s2">"authors"</span><span class="p">:</span> <span class="p">[</span><span class="n">extract_text</span><span class="p">(</span><span class="n">author</span><span class="p">)</span> <span class="k">for</span> <span class="n">author</span> <span class="ow">in</span> <span class="n">author_elements</span><span class="p">],</span>
|
||||||
|
<span class="s2">"publisher"</span><span class="p">:</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//a[@title="Publisher"]'</span><span class="p">),</span>
|
||||||
|
<span class="s2">"type"</span><span class="p">:</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "property__file")]//div[contains(@class, "property_value")]'</span><span class="p">),</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//img[contains(@class, "cover")]/@data-src'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">thumbnail</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">thumbnail</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'/'</span><span class="p">):</span>
|
||||||
|
<span class="n">result</span><span class="p">[</span><span class="s2">"thumbnail"</span><span class="p">]</span> <span class="o">=</span> <span class="n">thumbnail</span>
|
||||||
|
|
||||||
|
<span class="n">year</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "property_year")]//div[contains(@class, "property_value")]'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">year</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span><span class="p">[</span><span class="s2">"publishedDate"</span><span class="p">]</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">year</span><span class="p">,</span> <span class="s1">'%Y'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">content</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">language</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//div[contains(@class, "property_language")]//div[contains(@class, "property_value")]'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">language</span><span class="p">:</span>
|
||||||
|
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">i18n_language</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">language</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">book_rating</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//span[contains(@class, "book-rating-interest-score")]'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">book_rating</span> <span class="ow">and</span> <span class="nb">float</span><span class="p">(</span><span class="n">book_rating</span><span class="p">):</span>
|
||||||
|
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">i18n_book_rating</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">book_rating</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">file_quality</span> <span class="o">=</span> <span class="n">_text</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="s1">'.//span[contains(@class, "book-rating-quality-score")]'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">file_quality</span> <span class="ow">and</span> <span class="nb">float</span><span class="p">(</span><span class="n">file_quality</span><span class="p">):</span>
|
||||||
|
<span class="n">content</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">i18n_file_quality</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">file_quality</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">result</span><span class="p">[</span><span class="s2">"content"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" | "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">result</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="fetch_traits">
|
||||||
|
<a class="viewcode-back" href="../../../dev/engines/online/zlibrary.html#searx.engines.zlibrary.fetch_traits">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">fetch_traits</span><span class="p">(</span><span class="n">engine_traits</span><span class="p">:</span> <span class="n">EngineTraits</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fetch languages and other search arguments from zlibrary's search form."""</span>
|
||||||
|
<span class="c1"># pylint: disable=import-outside-toplevel, too-many-branches, too-many-statements</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.core</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">httpx</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span> <span class="c1"># see https://github.com/searxng/searxng/issues/762</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">language_tag</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_use_old_values</span><span class="p">():</span>
|
||||||
|
<span class="c1"># don't change anything, re-use the existing values</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">"z-library"</span><span class="p">][</span><span class="s2">"all_locale"</span><span class="p">]</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span> <span class="o">=</span> <span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">"z-library"</span><span class="p">][</span><span class="s2">"custom"</span><span class="p">]</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span> <span class="o">=</span> <span class="n">ENGINE_TRAITS</span><span class="p">[</span><span class="s2">"z-library"</span><span class="p">][</span><span class="s2">"languages"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">base_url</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="p">(</span><span class="n">SearxException</span><span class="p">,</span> <span class="n">httpx</span><span class="o">.</span><span class="n">HTTPError</span><span class="p">)</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"ERROR: zlibrary domain '</span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">' is seized?"</span><span class="p">)</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">" --> </span><span class="si">{</span><span class="n">exc</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">_use_old_values</span><span class="p">()</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resp</span><span class="o">.</span><span class="n">ok</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"Response from zlibrary's search page is not OK."</span><span class="p">)</span>
|
||||||
|
<span class="n">dom</span> <span class="o">=</span> <span class="n">html</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">domain_is_seized</span><span class="p">(</span><span class="n">dom</span><span class="p">):</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"ERROR: zlibrary domain is seized: </span><span class="si">{</span><span class="n">base_url</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="n">_use_old_values</span><span class="p">()</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">all_locale</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ext"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="n">l</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="c1"># years_from</span>
|
||||||
|
<span class="n">l</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">year</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[@id='advSearch-noJS']//select[@id='sf_yearFrom']/option"</span><span class="p">):</span>
|
||||||
|
<span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">year</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"year_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">l</span>
|
||||||
|
|
||||||
|
<span class="c1"># years_to</span>
|
||||||
|
<span class="n">l</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">year</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[@id='advSearch-noJS']//select[@id='sf_yearTo']/option"</span><span class="p">):</span>
|
||||||
|
<span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">year</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"year_to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">l</span>
|
||||||
|
|
||||||
|
<span class="c1"># ext (file extensions)</span>
|
||||||
|
<span class="n">l</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">ext</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[@id='advSearch-noJS']//select[@id='sf_extensions']/option"</span><span class="p">):</span>
|
||||||
|
<span class="n">l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ext</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span> <span class="ow">or</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">custom</span><span class="p">[</span><span class="s2">"ext"</span><span class="p">]</span> <span class="o">=</span> <span class="n">l</span>
|
||||||
|
|
||||||
|
<span class="c1"># Handle languages</span>
|
||||||
|
<span class="c1"># Z-library uses English names for languages, so we need to map them to their respective locales</span>
|
||||||
|
<span class="n">language_name_locale_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="k">for</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">localedata</span><span class="o">.</span><span class="n">locale_identifiers</span><span class="p">():</span>
|
||||||
|
<span class="c1"># Create a Locale object for the current locale</span>
|
||||||
|
<span class="n">loc</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">loc</span><span class="o">.</span><span class="n">english_name</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">language_name_locale_map</span><span class="p">[</span><span class="n">loc</span><span class="o">.</span><span class="n">english_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">loc</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">eval_xpath_list</span><span class="p">(</span><span class="n">dom</span><span class="p">,</span> <span class="s2">"//div[@id='advSearch-noJS']//select[@id='sf_languages']/option"</span><span class="p">):</span>
|
||||||
|
<span class="n">eng_lang</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"value"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">eng_lang</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">language_name_locale_map</span><span class="p">[</span><span class="n">eng_lang</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span>
|
||||||
|
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
||||||
|
<span class="c1"># silently ignore unknown languages</span>
|
||||||
|
<span class="c1"># print("ERROR: %s is unknown by babel" % (eng_lang))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="n">conflict</span> <span class="o">=</span> <span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sxng_lang</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">conflict</span> <span class="o">!=</span> <span class="n">eng_lang</span><span class="p">:</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"CONFLICT: babel </span><span class="si">%s</span><span class="s2"> --> </span><span class="si">%s</span><span class="s2">, </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">sxng_lang</span><span class="p">,</span> <span class="n">conflict</span><span class="p">,</span> <span class="n">eng_lang</span><span class="p">))</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">engine_traits</span><span class="o">.</span><span class="n">languages</span><span class="p">[</span><span class="n">sxng_lang</span><span class="p">]</span> <span class="o">=</span> <span class="n">eng_lang</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../engines.html">searx.engines</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
257
_modules/searx/exceptions.html
Normal file
@@ -0,0 +1,257 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.exceptions — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.exceptions</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.exceptions</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Exception types raised by SearXNG modules."""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml.etree</span><span class="w"> </span><span class="kn">import</span> <span class="n">XPath</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxException</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Base SearXNG exception."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxParameterException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxParameterException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxParameterException</span><span class="p">(</span><span class="n">SearxException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Raised when query miss a required parameter"""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">value</span> <span class="o">==</span> <span class="s1">''</span> <span class="ow">or</span> <span class="n">value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Empty </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2"> parameter"</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">message</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"Invalid value </span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2"> for parameter </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">message</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">parameter_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">name</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">parameter_value</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">value</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxSettingsException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxSettingsException">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxSettingsException</span><span class="p">(</span><span class="n">SearxException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Error while loading the settings"""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="ne">Exception</span><span class="p">,</span> <span class="n">filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">message</span> <span class="o">=</span> <span class="n">message</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxEngineException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineException</span><span class="p">(</span><span class="n">SearxException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Error inside an engine"""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxXPathSyntaxException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxXPathSyntaxException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxXPathSyntaxException</span><span class="p">(</span><span class="n">SearxEngineException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Syntax error in a XPATH"""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">:</span> <span class="s2">"str | XPath"</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span> <span class="o">+</span> <span class="s2">" "</span> <span class="o">+</span> <span class="n">message</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">message</span>
|
||||||
|
<span class="c1"># str(xpath_spec) to deal with str and XPath instance</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">xpath_str</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxEngineResponseException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineResponseException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineResponseException</span><span class="p">(</span><span class="n">SearxEngineException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Impossible to parse the result of an engine"""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxEngineAPIException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineAPIException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineAPIException</span><span class="p">(</span><span class="n">SearxEngineResponseException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The website has returned an application error"""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxEngineAccessDeniedException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineAccessDeniedException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineAccessDeniedException</span><span class="p">(</span><span class="n">SearxEngineResponseException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The website is blocking the access"""</span>
|
||||||
|
|
||||||
|
<span class="n">SUSPEND_TIME_SETTING</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"search.suspended_times.SearxEngineAccessDenied"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""This settings contains the default suspended time (default 86400 sec / 1</span>
|
||||||
|
<span class="sd"> day)."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">suspended_time</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">'Access denied'</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Generic exception to raise when an engine denies access to the results.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param suspended_time: How long the engine is going to be suspended in</span>
|
||||||
|
<span class="sd"> second. Defaults to None.</span>
|
||||||
|
<span class="sd"> :type suspended_time: int, None</span>
|
||||||
|
<span class="sd"> :param message: Internal message. Defaults to ``Access denied``</span>
|
||||||
|
<span class="sd"> :type message: str</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="n">suspended_time</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">suspended_time</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_default_suspended_time</span><span class="p">()</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">message</span><span class="si">}</span><span class="s2"> (suspended_time=</span><span class="si">{</span><span class="n">suspended_time</span><span class="si">}</span><span class="s2">)"</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">suspended_time</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">suspended_time</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_get_default_suspended_time</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_setting</span> <span class="c1"># pylint: disable=C0415</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">get_setting</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SUSPEND_TIME_SETTING</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxEngineCaptchaException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineCaptchaException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineCaptchaException</span><span class="p">(</span><span class="n">SearxEngineAccessDeniedException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The website has returned a CAPTCHA."""</span>
|
||||||
|
|
||||||
|
<span class="n">SUSPEND_TIME_SETTING</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"search.suspended_times.SearxEngineCaptcha"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""This settings contains the default suspended time (default 86400 sec / 1</span>
|
||||||
|
<span class="sd"> day)."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">suspended_time</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">'CAPTCHA'</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span> <span class="n">suspended_time</span><span class="o">=</span><span class="n">suspended_time</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxEngineTooManyRequestsException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineTooManyRequestsException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineTooManyRequestsException</span><span class="p">(</span><span class="n">SearxEngineAccessDeniedException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The website has returned a Too Many Request status code</span>
|
||||||
|
|
||||||
|
<span class="sd"> By default, SearXNG stops sending requests to this engine for 1 hour.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">SUSPEND_TIME_SETTING</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"search.suspended_times.SearxEngineTooManyRequests"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""This settings contains the default suspended time (default 3660 sec / 1</span>
|
||||||
|
<span class="sd"> hour)."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">suspended_time</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">'Too many request'</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span> <span class="n">suspended_time</span><span class="o">=</span><span class="n">suspended_time</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SearxEngineXPathException">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.exceptions.html#searx.exceptions.SearxEngineXPathException">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SearxEngineXPathException</span><span class="p">(</span><span class="n">SearxEngineResponseException</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Error while getting the result of an XPath expression"""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">xpath_spec</span><span class="p">:</span> <span class="s2">"str | XPath"</span><span class="p">,</span> <span class="n">message</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span> <span class="o">+</span> <span class="s2">" "</span> <span class="o">+</span> <span class="n">message</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">message</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">message</span>
|
||||||
|
<span class="c1"># str(xpath_spec) to deal with str and XPath instance</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">xpath_str</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">xpath_spec</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
196
_modules/searx/extended_types.html
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.extended_types — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.extended_types</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.extended_types</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""This module implements the type extensions applied by SearXNG.</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`flask.request` is replaced by :py:obj:`sxng_request`</span>
|
||||||
|
<span class="sd">- :py:obj:`flask.Request` is replaced by :py:obj:`SXNG_Request`</span>
|
||||||
|
<span class="sd">- :py:obj:`httpx.response` is replaced by :py:obj:`SXNG_Response`</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
|
||||||
|
<span class="sd">.. py:attribute:: sxng_request</span>
|
||||||
|
<span class="sd"> :type: SXNG_Request</span>
|
||||||
|
|
||||||
|
<span class="sd"> A replacement for :py:obj:`flask.request` with type cast :py:obj:`SXNG_Request`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: SXNG_Request</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: SXNG_Response</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"SXNG_Request"</span><span class="p">,</span> <span class="s2">"sxng_request"</span><span class="p">,</span> <span class="s2">"SXNG_Response"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">httpx</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">searx.preferences</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">searx.results</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search.processors</span><span class="w"> </span><span class="kn">import</span> <span class="n">OnlineParamTypes</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNG_Request">
|
||||||
|
<a class="viewcode-back" href="../../dev/extended_types.html#searx.extended_types.SXNG_Request">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNG_Request</span><span class="p">(</span><span class="n">flask</span><span class="o">.</span><span class="n">Request</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""SearXNG extends the class :py:obj:`flask.Request` with properties from</span>
|
||||||
|
<span class="sd"> *this* class definition, see type cast :py:obj:`sxng_request`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">user_plugins</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""list of searx.plugins.Plugin.id (the id of the plugins)"""</span>
|
||||||
|
|
||||||
|
<span class="n">preferences</span><span class="p">:</span> <span class="s2">"searx.preferences.Preferences"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The preferences of the request."""</span>
|
||||||
|
|
||||||
|
<span class="n">errors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A list of errors (translated text) added by :py:obj:`searx.webapp` in</span>
|
||||||
|
<span class="sd"> case of errors."""</span>
|
||||||
|
<span class="c1"># request.form is of type werkzeug.datastructures.ImmutableMultiDict</span>
|
||||||
|
<span class="c1"># form: dict[str, str]</span>
|
||||||
|
|
||||||
|
<span class="n">start_time</span><span class="p">:</span> <span class="nb">float</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Start time of the request, :py:obj:`timeit.default_timer` added by</span>
|
||||||
|
<span class="sd"> :py:obj:`searx.webapp` to calculate the total time of the request."""</span>
|
||||||
|
|
||||||
|
<span class="n">render_time</span><span class="p">:</span> <span class="nb">float</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Duration of the rendering, calculated and added by</span>
|
||||||
|
<span class="sd"> :py:obj:`searx.webapp`."""</span>
|
||||||
|
|
||||||
|
<span class="n">timings</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="s2">"searx.results.Timing"</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A list of :py:obj:`searx.results.Timing` of the engines, calculatid in</span>
|
||||||
|
<span class="sd"> and hold by :py:obj:`searx.results.ResultContainer.timings`."""</span>
|
||||||
|
|
||||||
|
<span class="n">remote_addr</span><span class="p">:</span> <span class="nb">str</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1">#: A replacement for :py:obj:`flask.request` with type cast :py:`SXNG_Request`.</span>
|
||||||
|
<span class="n">sxng_request</span> <span class="o">=</span> <span class="n">typing</span><span class="o">.</span><span class="n">cast</span><span class="p">(</span><span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNG_Response">
|
||||||
|
<a class="viewcode-back" href="../../dev/extended_types.html#searx.extended_types.SXNG_Response">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNG_Response</span><span class="p">(</span><span class="n">httpx</span><span class="o">.</span><span class="n">Response</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""SearXNG extends the class :py:obj:`httpx.Response` with properties from</span>
|
||||||
|
<span class="sd"> *this* class (type cast of :py:obj:`httpx.Response`).</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> response = httpx.get("https://example.org")</span>
|
||||||
|
<span class="sd"> response = typing.cast(SXNG_Response, response)</span>
|
||||||
|
<span class="sd"> if response.ok:</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> query_was = search_params["query"]</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">ok</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
|
<span class="n">search_params</span><span class="p">:</span> <span class="s2">"OnlineParamTypes"</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
663
_modules/searx/favicons/cache.html
Normal file
@@ -0,0 +1,663 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.favicons.cache — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.favicons.cache</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.favicons.cache</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Implementations for caching favicons.</span>
|
||||||
|
|
||||||
|
<span class="sd">:py:obj:`FaviconCacheConfig`:</span>
|
||||||
|
<span class="sd"> Configuration of the favicon cache</span>
|
||||||
|
|
||||||
|
<span class="sd">:py:obj:`FaviconCache`:</span>
|
||||||
|
<span class="sd"> Abstract base class for the implementation of a favicon cache.</span>
|
||||||
|
|
||||||
|
<span class="sd">:py:obj:`FaviconCacheSQLite`:</span>
|
||||||
|
<span class="sd"> Favicon cache that manages the favicon BLOBs in a SQLite DB.</span>
|
||||||
|
|
||||||
|
<span class="sd">:py:obj:`FaviconCacheNull`:</span>
|
||||||
|
<span class="sd"> Fallback solution if the configured cache cannot be used for system reasons.</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">dataclasses</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">sqlite3</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typer</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">sqlitedb</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">humanize_bytes</span><span class="p">,</span> <span class="n">humanize_number</span>
|
||||||
|
|
||||||
|
<span class="n">CACHE</span><span class="p">:</span> <span class="s2">"FaviconCache"</span>
|
||||||
|
<span class="n">FALLBACK_ICON</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"FALLBACK_ICON"</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'favicons.cache'</span><span class="p">)</span>
|
||||||
|
<span class="n">app</span> <span class="o">=</span> <span class="n">typer</span><span class="o">.</span><span class="n">Typer</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="state">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.state">[docs]</a>
|
||||||
|
<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">():</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""show state of the cache"""</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span><span class="o">.</span><span class="n">report</span><span class="p">())</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="maintenance">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.maintenance">[docs]</a>
|
||||||
|
<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">debug</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""perform maintenance of the cache"""</span>
|
||||||
|
<span class="n">root_log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">debug</span><span class="p">:</span>
|
||||||
|
<span class="n">root_log</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">root_log</span><span class="o">.</span><span class="n">handlers</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">handler</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">StreamHandler</span><span class="p">()</span>
|
||||||
|
<span class="n">handler</span><span class="o">.</span><span class="n">setFormatter</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">Formatter</span><span class="p">(</span><span class="s2">"</span><span class="si">%(message)s</span><span class="s2">"</span><span class="p">))</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">handler</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">state_t0</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span>
|
||||||
|
<span class="n">CACHE</span><span class="o">.</span><span class="n">maintenance</span><span class="p">(</span><span class="n">force</span><span class="o">=</span><span class="n">force</span><span class="p">)</span>
|
||||||
|
<span class="n">state_t1</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span>
|
||||||
|
<span class="n">state_delta</span> <span class="o">=</span> <span class="n">state_t0</span> <span class="o">-</span> <span class="n">state_t1</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"The cache has been reduced by:"</span><span class="p">)</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="n">state_delta</span><span class="o">.</span><span class="n">report</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">- </span><span class="si">{descr}</span><span class="s2">: </span><span class="si">{val}</span><span class="s2">"</span><span class="p">)</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">))</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="init">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="s2">"FaviconCacheConfig"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of a global ``CACHE``"""</span>
|
||||||
|
|
||||||
|
<span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
<span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span> <span class="o">==</span> <span class="s2">"sqlite"</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">sqlite_version_info</span> <span class="o"><=</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">35</span><span class="p">):</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span>
|
||||||
|
<span class="s2">"Disable favicon caching completely: SQLite library (</span><span class="si">%s</span><span class="s2">) is too old! (require >= 3.35)"</span><span class="p">,</span>
|
||||||
|
<span class="n">sqlite3</span><span class="o">.</span><span class="n">sqlite_version</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheNull</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheSQLite</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">elif</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span> <span class="o">==</span> <span class="s2">"mem"</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Favicons are cached in memory, don't use this in production!"</span><span class="p">)</span>
|
||||||
|
<span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheMEM</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"favicons db_type '</span><span class="si">{</span><span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span><span class="si">}</span><span class="s2">' is unknown"</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheConfig">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheConfig">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Configuration of the favicon cache."""</span>
|
||||||
|
|
||||||
|
<span class="n">db_type</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"sqlite"</span><span class="p">,</span> <span class="s2">"mem"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"sqlite"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Type of the database:</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``sqlite``:</span>
|
||||||
|
<span class="sd"> :py:obj:`.cache.FaviconCacheSQLite`</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``mem``:</span>
|
||||||
|
<span class="sd"> :py:obj:`.cache.FaviconCacheMEM` (not recommended)</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">db_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">gettempdir</span><span class="p">()</span> <span class="o">+</span> <span class="n">os</span><span class="o">.</span><span class="n">sep</span> <span class="o">+</span> <span class="s2">"faviconcache.db"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL of the SQLite DB, the path to the database file."""</span>
|
||||||
|
|
||||||
|
<span class="n">HOLD_TIME</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">30</span> <span class="c1"># 30 days</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Hold time (default in sec.), after which a BLOB is removed from the cache."""</span>
|
||||||
|
|
||||||
|
<span class="n">LIMIT_TOTAL_BYTES</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">50</span> <span class="c1"># 50 MB</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Maximum of bytes (default) stored in the cache of all blobs. Note: The</span>
|
||||||
|
<span class="sd"> limit is only reached at each maintenance interval after which the oldest</span>
|
||||||
|
<span class="sd"> BLOBs are deleted; the limit is exceeded during the maintenance period. If</span>
|
||||||
|
<span class="sd"> the maintenance period is *too long* or maintenance is switched off</span>
|
||||||
|
<span class="sd"> completely, the cache grows uncontrollably."""</span>
|
||||||
|
|
||||||
|
<span class="n">BLOB_MAX_BYTES</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">20</span> <span class="c1"># 20 KB</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The maximum BLOB size in bytes that a favicon may have so that it can be</span>
|
||||||
|
<span class="sd"> saved in the cache. If the favicon is larger, it is not saved in the cache</span>
|
||||||
|
<span class="sd"> and must be requested by the client via the proxy."""</span>
|
||||||
|
|
||||||
|
<span class="n">MAINTENANCE_PERIOD</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to</span>
|
||||||
|
<span class="sd"> ``auto``."""</span>
|
||||||
|
|
||||||
|
<span class="n">MAINTENANCE_MODE</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"auto"</span><span class="p">,</span> <span class="s2">"off"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"auto"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Type of maintenance mode</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``auto``:</span>
|
||||||
|
<span class="sd"> Maintenance is carried out automatically as part of the maintenance</span>
|
||||||
|
<span class="sd"> intervals (:py:obj:`MAINTENANCE_PERIOD`); no external process is required.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``off``:</span>
|
||||||
|
<span class="sd"> Maintenance is switched off and must be carried out by an external process</span>
|
||||||
|
<span class="sd"> if required.</span>
|
||||||
|
<span class="sd"> """</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheStats">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheStats">[docs]</a>
|
||||||
|
<span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheStats</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Dataclass which provides information on the status of the cache."""</span>
|
||||||
|
|
||||||
|
<span class="n">favicons</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="nb">bytes</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">domains</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">resolvers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">field_descr</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="nb">type</span><span class="p">],</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="p">(</span><span class="s2">"favicons"</span><span class="p">,</span> <span class="s2">"number of favicons in cache"</span><span class="p">,</span> <span class="n">humanize_number</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"bytes"</span><span class="p">,</span> <span class="s2">"total size (approx. bytes) of cache"</span><span class="p">,</span> <span class="n">humanize_bytes</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"domains"</span><span class="p">,</span> <span class="s2">"total number of domains in cache"</span><span class="p">,</span> <span class="n">humanize_number</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"resolvers"</span><span class="p">,</span> <span class="s2">"number of resolvers"</span><span class="p">,</span> <span class="nb">str</span><span class="p">),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__sub__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="s2">"FaviconCacheStats"</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"FaviconCacheStats"</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"unsupported operand type(s) for +: '</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="si">}</span><span class="s2">' and '</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span><span class="p">)</span>
|
||||||
|
<span class="n">kwargs</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="k">for</span> <span class="n">field</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">field_descr</span><span class="p">:</span>
|
||||||
|
<span class="n">self_val</span><span class="p">,</span> <span class="n">other_val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field</span><span class="p">),</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">field</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="kc">None</span> <span class="ow">in</span> <span class="p">(</span><span class="n">self_val</span><span class="p">,</span> <span class="n">other_val</span><span class="p">):</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">self_val</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
|
||||||
|
<span class="n">kwargs</span><span class="p">[</span><span class="n">field</span><span class="p">]</span> <span class="o">=</span> <span class="n">self_val</span> <span class="o">-</span> <span class="n">other_val</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">kwargs</span><span class="p">[</span><span class="n">field</span><span class="p">]</span> <span class="o">=</span> <span class="n">self_val</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">report</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fmt</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{descr}</span><span class="s2">: </span><span class="si">{val}</span><span class="se">\n</span><span class="s2">"</span><span class="p">):</span>
|
||||||
|
<span class="n">s</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">field</span><span class="p">,</span> <span class="n">descr</span><span class="p">,</span> <span class="n">cast</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">field_descr</span><span class="p">:</span>
|
||||||
|
<span class="n">val</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="s2">"--"</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">s</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fmt</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">descr</span><span class="o">=</span><span class="n">descr</span><span class="p">,</span> <span class="n">val</span><span class="o">=</span><span class="n">val</span><span class="p">))</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">s</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCache">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCache</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Abstract base class for the implementation of a favicon cache."""</span>
|
||||||
|
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""An instance of the favicon cache is build up from the configuration."""</span>
|
||||||
|
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns ``None`` or the tuple of ``(data, mime)`` that has been</span>
|
||||||
|
<span class="sd"> registered in the cache. The ``None`` indicates that there was no entry</span>
|
||||||
|
<span class="sd"> in the cache."""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCache.set">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.set">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Set data and mime-type in the cache. If data is None, the</span>
|
||||||
|
<span class="sd"> :py:obj:`FALLBACK_ICON` is registered. in the cache."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCache.state">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.state">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">FaviconCacheStats</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a :py:obj:`FaviconCacheStats` (key/values) with information</span>
|
||||||
|
<span class="sd"> on the state of the cache."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCache.maintenance">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.maintenance">[docs]</a>
|
||||||
|
<span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Performs maintenance on the cache"""</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheNull">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheNull</span><span class="p">(</span><span class="n">FaviconCache</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A dummy favicon cache that caches nothing / a fallback solution. The</span>
|
||||||
|
<span class="sd"> NullCache is used when more efficient caches such as the</span>
|
||||||
|
<span class="sd"> :py:obj:`FaviconCacheSQLite` cannot be used because, for example, the SQLite</span>
|
||||||
|
<span class="sd"> library is only available in an old version and does not meet the</span>
|
||||||
|
<span class="sd"> requirements."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheNull.set">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.set">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheNull.state">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.state">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span><span class="n">favicons</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheNull.maintenance">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.maintenance">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||||||
|
<span class="k">pass</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheSQLite">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheSQLite</span><span class="p">(</span><span class="n">sqlitedb</span><span class="o">.</span><span class="n">SQLiteAppl</span><span class="p">,</span> <span class="n">FaviconCache</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportUnsafeMultipleInheritance]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Favicon cache that manages the favicon BLOBs in a SQLite DB. The DB</span>
|
||||||
|
<span class="sd"> model in the SQLite DB is implemented using the abstract class</span>
|
||||||
|
<span class="sd"> :py:obj:`sqlitedb.SQLiteAppl`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> For introspection of the DB, jump into developer environment and run command</span>
|
||||||
|
<span class="sd"> to show cache state::</span>
|
||||||
|
|
||||||
|
<span class="sd"> $ ./manage pyenv.cmd bash --norc --noprofile</span>
|
||||||
|
<span class="sd"> (py3) python -m searx.favicons cache state</span>
|
||||||
|
|
||||||
|
<span class="sd"> The following configurations are required / supported:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - :py:obj:`FaviconCacheConfig.db_url`</span>
|
||||||
|
<span class="sd"> - :py:obj:`FaviconCacheConfig.HOLD_TIME`</span>
|
||||||
|
<span class="sd"> - :py:obj:`FaviconCacheConfig.LIMIT_TOTAL_BYTES`</span>
|
||||||
|
<span class="sd"> - :py:obj:`FaviconCacheConfig.BLOB_MAX_BYTES`</span>
|
||||||
|
<span class="sd"> - :py:obj:`MAINTENANCE_PERIOD`</span>
|
||||||
|
<span class="sd"> - :py:obj:`MAINTENANCE_MODE`</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">DB_SCHEMA</span> <span class="o">=</span> <span class="mi">1</span>
|
||||||
|
|
||||||
|
<span class="n">DDL_BLOBS</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span>
|
||||||
|
<span class="s2">CREATE TABLE IF NOT EXISTS blobs (</span>
|
||||||
|
<span class="s2"> sha256 TEXT,</span>
|
||||||
|
<span class="s2"> bytes_c INTEGER,</span>
|
||||||
|
<span class="s2"> mime TEXT NOT NULL,</span>
|
||||||
|
<span class="s2"> data BLOB NOT NULL,</span>
|
||||||
|
<span class="s2"> PRIMARY KEY (sha256))"""</span>
|
||||||
|
|
||||||
|
<span class="w"> </span><span class="sd">"""Table to store BLOB objects by their sha256 hash values."""</span>
|
||||||
|
|
||||||
|
<span class="n">DDL_BLOB_MAP</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span>
|
||||||
|
<span class="s2">CREATE TABLE IF NOT EXISTS blob_map (</span>
|
||||||
|
<span class="s2"> m_time INTEGER DEFAULT (strftime('</span><span class="si">%s</span><span class="s2">', 'now')), -- last modified (unix epoch) time in sec.</span>
|
||||||
|
<span class="s2"> sha256 TEXT,</span>
|
||||||
|
<span class="s2"> resolver TEXT,</span>
|
||||||
|
<span class="s2"> authority TEXT,</span>
|
||||||
|
<span class="s2"> PRIMARY KEY (resolver, authority))"""</span>
|
||||||
|
|
||||||
|
<span class="w"> </span><span class="sd">"""Table to map from (resolver, authority) to sha256 hash values."""</span>
|
||||||
|
|
||||||
|
<span class="n">DDL_CREATE_TABLES</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"blobs"</span><span class="p">:</span> <span class="n">DDL_BLOBS</span><span class="p">,</span>
|
||||||
|
<span class="s2">"blob_map"</span><span class="p">:</span> <span class="n">DDL_BLOB_MAP</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="n">SQL_DROP_LEFTOVER_BLOBS</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s2">"DELETE FROM blobs WHERE sha256 IN ("</span>
|
||||||
|
<span class="s2">" SELECT b.sha256"</span>
|
||||||
|
<span class="s2">" FROM blobs b"</span>
|
||||||
|
<span class="s2">" LEFT JOIN blob_map bm"</span>
|
||||||
|
<span class="s2">" ON b.sha256 = bm.sha256"</span>
|
||||||
|
<span class="s2">" WHERE bm.sha256 IS NULL)"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Delete blobs.sha256 (BLOBs) no longer in blob_map.sha256."""</span>
|
||||||
|
|
||||||
|
<span class="n">SQL_ITER_BLOBS_SHA256_BYTES_C</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s2">"SELECT b.sha256, b.bytes_c FROM blobs b"</span>
|
||||||
|
<span class="s2">" JOIN blob_map bm "</span>
|
||||||
|
<span class="s2">" ON b.sha256 = bm.sha256"</span>
|
||||||
|
<span class="s2">" ORDER BY bm.m_time ASC"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">SQL_INSERT_BLOBS</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s2">"INSERT INTO blobs (sha256, bytes_c, mime, data) VALUES (?, ?, ?, ?)"</span>
|
||||||
|
<span class="s2">" ON CONFLICT (sha256) DO NOTHING"</span>
|
||||||
|
<span class="p">)</span> <span class="c1"># fmt: skip</span>
|
||||||
|
|
||||||
|
<span class="n">SQL_INSERT_BLOB_MAP</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="s2">"INSERT INTO blob_map (sha256, resolver, authority) VALUES (?, ?, ?)"</span>
|
||||||
|
<span class="s2">" ON CONFLICT DO UPDATE "</span>
|
||||||
|
<span class="s2">" SET sha256=excluded.sha256, m_time=strftime('</span><span class="si">%s</span><span class="s2">', 'now')"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""An instance of the favicon cache is build up from the configuration."""</span> <span class="c1">#</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span> <span class="o">==</span> <span class="s2">":memory:"</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="s2">"don't use SQLite DB in :memory: in production!!"</span><span class="p">)</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="n">sql</span> <span class="o">=</span> <span class="s2">"SELECT sha256 FROM blob_map WHERE resolver = ? AND authority = ?"</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">))</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">res</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">sha256</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">sha256</span> <span class="o">==</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span>
|
||||||
|
|
||||||
|
<span class="n">sql</span> <span class="o">=</span> <span class="s2">"SELECT data, mime FROM blobs WHERE sha256 = ?"</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,))</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">res</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">res</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheSQLite.set">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.set">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAINTENANCE_MODE</span> <span class="o">==</span> <span class="s2">"auto"</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">next_maintenance_time</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Should automatic maintenance be moved to a new thread?</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
|
||||||
|
<span class="s2">"favicon resolver </span><span class="si">%s</span><span class="s2"> tries to cache mime-type None for authority </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="n">resolver</span><span class="p">,</span>
|
||||||
|
<span class="n">authority</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">bytes_c</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span> <span class="ow">or</span> <span class="sa">b</span><span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">bytes_c</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">BLOB_MAX_BYTES</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||||
|
<span class="s2">"favicon of resolver: </span><span class="si">%s</span><span class="s2"> / authority: </span><span class="si">%s</span><span class="s2"> to big to cache (bytes: </span><span class="si">%s</span><span class="s2">) "</span> <span class="o">%</span> <span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">,</span> <span class="n">bytes_c</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">sha256</span> <span class="o">=</span> <span class="n">FALLBACK_ICON</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">sha256</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">sha256</span> <span class="o">!=</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_INSERT_BLOBS</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,</span> <span class="n">bytes_c</span><span class="p">,</span> <span class="n">mime</span><span class="p">,</span> <span class="n">data</span><span class="p">))</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_INSERT_BLOB_MAP</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,</span> <span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">))</span>
|
||||||
|
<span class="c1"># hint: the with context of the connection object closes the transaction</span>
|
||||||
|
<span class="c1"># but not the DB connection. The connection has to be closed by the</span>
|
||||||
|
<span class="c1"># caller of self.connect()!</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="nd">@property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">next_maintenance_time</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns (unix epoch) time of the next maintenance."""</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAINTENANCE_PERIOD</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">m_time</span><span class="p">(</span><span class="s2">"LAST_MAINTENANCE"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheSQLite.maintenance">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.maintenance">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="c1"># Prevent parallel DB maintenance cycles from other DB connections</span>
|
||||||
|
<span class="c1"># (e.g. in multi thread or process environments).</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">force</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_maintenance_time</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"no maintenance required yet, next maintenance interval is in the future"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">"LAST_MAINTENANCE"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span> <span class="c1"># hint: this (also) sets the m_time of the property!</span>
|
||||||
|
|
||||||
|
<span class="c1"># Do maintenance tasks. This can be take a little more time, to avoid</span>
|
||||||
|
<span class="c1"># DB locks, establish a new DB connection.</span>
|
||||||
|
|
||||||
|
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="c1"># drop items not in HOLD time</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"DELETE FROM blob_map"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" WHERE cast(m_time as integer) < cast(strftime('%s', 'now') as integer) - </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">HOLD_TIME</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"dropped </span><span class="si">%s</span><span class="s2"> obsolete blob_map items from db"</span><span class="p">,</span> <span class="n">res</span><span class="o">.</span><span class="n">rowcount</span><span class="p">)</span>
|
||||||
|
<span class="n">res</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_DROP_LEFTOVER_BLOBS</span><span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"dropped </span><span class="si">%s</span><span class="s2"> obsolete BLOBS from db"</span><span class="p">,</span> <span class="n">res</span><span class="o">.</span><span class="n">rowcount</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># drop old items to be in LIMIT_TOTAL_BYTES</span>
|
||||||
|
<span class="n">total_bytes</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"SELECT SUM(bytes_c) FROM blobs"</span><span class="p">)</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span> <span class="ow">or</span> <span class="mi">0</span>
|
||||||
|
<span class="k">if</span> <span class="n">total_bytes</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">LIMIT_TOTAL_BYTES</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">x</span> <span class="o">=</span> <span class="n">total_bytes</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">LIMIT_TOTAL_BYTES</span>
|
||||||
|
<span class="n">c</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="n">sha_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_ITER_BLOBS_SHA256_BYTES_C</span><span class="p">):</span>
|
||||||
|
<span class="n">sha256</span><span class="p">,</span> <span class="n">bytes_c</span> <span class="o">=</span> <span class="n">row</span>
|
||||||
|
<span class="n">sha_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">sha256</span><span class="p">)</span>
|
||||||
|
<span class="n">c</span> <span class="o">+=</span> <span class="n">bytes_c</span>
|
||||||
|
<span class="k">if</span> <span class="n">c</span> <span class="o">></span> <span class="n">x</span><span class="p">:</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
<span class="k">if</span> <span class="n">sha_list</span><span class="p">:</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"DELETE FROM blobs WHERE sha256 IN ('</span><span class="si">%s</span><span class="s2">')"</span> <span class="o">%</span> <span class="s2">"','"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">sha_list</span><span class="p">))</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"DELETE FROM blob_map WHERE sha256 IN ('</span><span class="si">%s</span><span class="s2">')"</span> <span class="o">%</span> <span class="s2">"','"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">sha_list</span><span class="p">))</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"dropped </span><span class="si">%s</span><span class="s2"> blobs with total size of </span><span class="si">%s</span><span class="s2"> bytes"</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">sha_list</span><span class="p">),</span> <span class="n">c</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Vacuuming the WALs</span>
|
||||||
|
<span class="c1"># https://www.theunterminatedstring.com/sqlite-vacuuming/</span>
|
||||||
|
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"PRAGMA wal_checkpoint(TRUNCATE)"</span><span class="p">)</span>
|
||||||
|
<span class="n">conn</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_query_val</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">val</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">default</span>
|
||||||
|
<span class="k">return</span> <span class="n">val</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheSQLite.state">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.state">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">FaviconCacheStats</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span>
|
||||||
|
<span class="n">favicons</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">"SELECT count(*) FROM blobs"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
|
||||||
|
<span class="nb">bytes</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">"SELECT SUM(bytes_c) FROM blobs"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
|
||||||
|
<span class="n">domains</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">"SELECT count(*) FROM (SELECT authority FROM blob_map GROUP BY authority)"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
|
||||||
|
<span class="n">resolvers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">"SELECT count(*) FROM (SELECT resolver FROM blob_map GROUP BY resolver)"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheMEM">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconCacheMEM</span><span class="p">(</span><span class="n">FaviconCache</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Favicon cache in process' memory. Its just a POC that stores the</span>
|
||||||
|
<span class="sd"> favicons in the memory of the process.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. attention::</span>
|
||||||
|
|
||||||
|
<span class="sd"> Don't use it in production, it will blow up your memory!!</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="n">sha</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">resolver</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">authority</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">sha</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sha</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">data</span> <span class="o">==</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheMEM.set">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.set">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">FALLBACK_ICON</span>
|
||||||
|
<span class="n">mime</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">elif</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
|
||||||
|
<span class="s2">"favicon resolver </span><span class="si">%s</span><span class="s2"> tries to cache mime-type None for authority </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="n">resolver</span><span class="p">,</span>
|
||||||
|
<span class="n">authority</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">digest</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="p">[</span><span class="n">digest</span><span class="p">]</span> <span class="o">=</span> <span class="n">data</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span><span class="p">[</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">resolver</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">authority</span><span class="si">}</span><span class="s2">"</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">digest</span><span class="p">,</span> <span class="n">mime</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheMEM.state">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.state">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span><span class="n">favicons</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconCacheMEM.maintenance">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.maintenance">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||||||
|
<span class="k">pass</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
174
_modules/searx/favicons/config.html
Normal file
@@ -0,0 +1,174 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.favicons.config — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.favicons.config</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.favicons.config</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.cache</span><span class="w"> </span><span class="kn">import</span> <span class="n">FaviconCacheConfig</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.proxy</span><span class="w"> </span><span class="kn">import</span> <span class="n">FaviconProxyConfig</span>
|
||||||
|
|
||||||
|
<span class="n">CONFIG_SCHEMA</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span>
|
||||||
|
<span class="sd">"""Version of the configuration schema."""</span>
|
||||||
|
|
||||||
|
<span class="n">TOML_CACHE_CFG</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="s2">"FaviconConfig"</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="sd">"""Cache config objects by TOML's filename."""</span>
|
||||||
|
|
||||||
|
<span class="n">DEFAULT_CFG_TOML_PATH</span> <span class="o">=</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span class="n">parent</span> <span class="o">/</span> <span class="s2">"favicons.toml"</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconConfig">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.config.FaviconConfig">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The class aggregates configurations of the favicon tools"""</span>
|
||||||
|
|
||||||
|
<span class="n">cfg_schema</span><span class="p">:</span> <span class="nb">int</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Config's schema version. The specification of the version of the schema</span>
|
||||||
|
<span class="sd"> is mandatory, currently only version :py:obj:`CONFIG_SCHEMA` is supported.</span>
|
||||||
|
<span class="sd"> By specifying a version, it is possible to ensure downward compatibility in</span>
|
||||||
|
<span class="sd"> the event of future changes to the configuration schema"""</span>
|
||||||
|
|
||||||
|
<span class="n">cache</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">FaviconCacheConfig</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Setup of the :py:obj:`.cache.FaviconCacheConfig`."""</span>
|
||||||
|
|
||||||
|
<span class="n">proxy</span><span class="p">:</span> <span class="n">FaviconProxyConfig</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">FaviconProxyConfig</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Setup of the :py:obj:`.proxy.FaviconProxyConfig`."""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconConfig.from_toml_file">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.config.FaviconConfig.from_toml_file">[docs]</a>
|
||||||
|
<span class="nd">@classmethod</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">from_toml_file</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">,</span> <span class="n">use_cache</span><span class="p">:</span> <span class="nb">bool</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"FaviconConfig"</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Create a config object from a TOML file, the ``use_cache`` argument</span>
|
||||||
|
<span class="sd"> specifies whether a cache should be used.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">cached</span> <span class="o">=</span> <span class="n">TOML_CACHE_CFG</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">cfg_file</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">use_cache</span> <span class="ow">and</span> <span class="n">cached</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">cached</span>
|
||||||
|
|
||||||
|
<span class="k">with</span> <span class="n">cfg_file</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">toml</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">_FaviconConfig</span><span class="p">)</span>
|
||||||
|
<span class="n">schema</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">favicons</span><span class="o">.</span><span class="n">cfg_schema</span>
|
||||||
|
<span class="k">if</span> <span class="n">schema</span> <span class="o">!=</span> <span class="n">CONFIG_SCHEMA</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"config schema version </span><span class="si">{</span><span class="n">CONFIG_SCHEMA</span><span class="si">}</span><span class="s2"> is needed, version </span><span class="si">{</span><span class="n">schema</span><span class="si">}</span><span class="s2"> is given in </span><span class="si">{</span><span class="n">cfg_file</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">favicons</span>
|
||||||
|
<span class="k">if</span> <span class="n">use_cache</span> <span class="ow">and</span> <span class="n">cached</span><span class="p">:</span>
|
||||||
|
<span class="n">TOML_CACHE_CFG</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">cfg_file</span><span class="o">.</span><span class="n">resolve</span><span class="p">())]</span> <span class="o">=</span> <span class="n">cfg</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">cfg</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">_FaviconConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
<span class="c1"># wrapper struct for root object "favicons."</span>
|
||||||
|
<span class="n">favicons</span><span class="p">:</span> <span class="n">FaviconConfig</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
362
_modules/searx/favicons/proxy.html
Normal file
@@ -0,0 +1,362 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.favicons.proxy — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.favicons.proxy</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.favicons.proxy</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Implementations for a favicon proxy"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">importlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">pathlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib.parse</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">httpx</span><span class="w"> </span><span class="kn">import</span> <span class="n">HTTPError</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_setting</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.webutils</span><span class="w"> </span><span class="kn">import</span> <span class="n">new_hmac</span><span class="p">,</span> <span class="n">is_hmac_of</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.exceptions</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearxEngineResponseException</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">sxng_request</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.resolvers</span><span class="w"> </span><span class="kn">import</span> <span class="n">DEFAULT_RESOLVER_MAP</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">cache</span>
|
||||||
|
|
||||||
|
<span class="n">DEFAULT_FAVICON_URL</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">CFG</span><span class="p">:</span> <span class="s2">"FaviconProxyConfig"</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="s2">"FaviconProxyConfig"</span><span class="p">):</span>
|
||||||
|
<span class="k">global</span> <span class="n">CFG</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
<span class="n">CFG</span> <span class="o">=</span> <span class="n">cfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_initial_resolver_map</span><span class="p">():</span>
|
||||||
|
<span class="n">d</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"search.favicon_resolver"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">if</span> <span class="n">name</span><span class="p">:</span>
|
||||||
|
<span class="n">func</span> <span class="o">=</span> <span class="n">DEFAULT_RESOLVER_MAP</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">func</span><span class="p">:</span>
|
||||||
|
<span class="n">d</span> <span class="o">=</span> <span class="p">{</span><span class="n">name</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"searx.favicons.resolvers.</span><span class="si">{</span><span class="n">func</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">"</span><span class="p">}</span>
|
||||||
|
<span class="k">return</span> <span class="n">d</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconProxyConfig">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">FaviconProxyConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Configuration of the favicon proxy."""</span>
|
||||||
|
|
||||||
|
<span class="n">max_age</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span> <span class="c1"># seven days</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""HTTP header Cache-Control_ ``max-age``</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _Cache-Control: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">secret_key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"server.secret_key"</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""By default, the value from :ref:`server.secret_key <settings server>`</span>
|
||||||
|
<span class="sd"> setting is used."""</span>
|
||||||
|
|
||||||
|
<span class="n">resolver_timeout</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"outgoing.request_timeout"</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Timeout which the resolvers should not exceed, is usually passed to the</span>
|
||||||
|
<span class="sd"> outgoing request of the resolver. By default, the value from</span>
|
||||||
|
<span class="sd"> :ref:`outgoing.request_timeout <settings outgoing>` setting is used."""</span>
|
||||||
|
|
||||||
|
<span class="n">resolver_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">_initial_resolver_map</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The resolver_map is a key / value dictionary where the key is the name of</span>
|
||||||
|
<span class="sd"> the resolver and the value is the fully qualifying name (fqn) of resolver's</span>
|
||||||
|
<span class="sd"> function (the callable). The resolvers from the python module</span>
|
||||||
|
<span class="sd"> :py:obj:`searx.favicons.resolver` are available by default."""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconProxyConfig.get_resolver">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.get_resolver">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_resolver</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Callable</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns the callable object (function) of the resolver with the</span>
|
||||||
|
<span class="sd"> ``name``. If no resolver is registered for the ``name``, ``None`` is</span>
|
||||||
|
<span class="sd"> returned.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">fqn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">fqn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
<span class="n">mod_name</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">func_name</span> <span class="o">=</span> <span class="n">fqn</span><span class="o">.</span><span class="n">rpartition</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
|
||||||
|
<span class="n">mod</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">mod_name</span><span class="p">)</span>
|
||||||
|
<span class="n">func</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">func_name</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"resolver </span><span class="si">{</span><span class="n">fqn</span><span class="si">}</span><span class="s2"> is not implemented"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">func</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">favicon_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"ui.static_path"</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"/themes/</span><span class="si">{theme}</span><span class="s2">/img/empty_favicon.svg"</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">favicon_mime_type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"image/svg+xml"</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconProxyConfig.favicon">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.favicon">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">favicon</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">replacements</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns pathname and mimetype of the default favicon."""</span>
|
||||||
|
<span class="k">return</span> <span class="p">(</span>
|
||||||
|
<span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">favicon_path</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">replacements</span><span class="p">)),</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">favicon_mime_type</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="FaviconProxyConfig.favicon_data_url">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.favicon_data_url">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">favicon_data_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">replacements</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns data image URL of the default favicon."""</span>
|
||||||
|
|
||||||
|
<span class="n">cache_key</span> <span class="o">=</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">x</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">replacements</span><span class="p">[</span><span class="n">x</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">replacements</span><span class="o">.</span><span class="n">keys</span><span class="p">()),</span> <span class="n">key</span><span class="o">=</span><span class="nb">str</span><span class="p">))</span>
|
||||||
|
<span class="n">data_url</span> <span class="o">=</span> <span class="n">DEFAULT_FAVICON_URL</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cache_key</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">data_url</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">data_url</span>
|
||||||
|
|
||||||
|
<span class="n">fav</span><span class="p">,</span> <span class="n">mimetype</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon</span><span class="p">(</span><span class="o">**</span><span class="n">replacements</span><span class="p">)</span>
|
||||||
|
<span class="c1"># hint: encoding utf-8 limits favicons to be a SVG image</span>
|
||||||
|
<span class="k">with</span> <span class="n">fav</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="s2">"r"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||||
|
<span class="n">data_url</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">data_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">data_url</span><span class="p">)</span>
|
||||||
|
<span class="n">data_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"data:</span><span class="si">{</span><span class="n">mimetype</span><span class="si">}</span><span class="s2">;utf8,</span><span class="si">{</span><span class="n">data_url</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">DEFAULT_FAVICON_URL</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">data_url</span>
|
||||||
|
<span class="k">return</span> <span class="n">data_url</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="favicon_proxy">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.favicon_proxy">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">favicon_proxy</span><span class="p">():</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""REST API of SearXNG's favicon proxy service</span>
|
||||||
|
|
||||||
|
<span class="sd"> ::</span>
|
||||||
|
|
||||||
|
<span class="sd"> /favicon_proxy?authority=<...>&h=<...></span>
|
||||||
|
|
||||||
|
<span class="sd"> ``authority``:</span>
|
||||||
|
<span class="sd"> Domain name :rfc:`3986` / see :py:obj:`favicon_url`</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``h``:</span>
|
||||||
|
<span class="sd"> HMAC :rfc:`2104`, build up from the :ref:`server.secret_key <settings</span>
|
||||||
|
<span class="sd"> server>` setting.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">authority</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'authority'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># malformed request or RFC 3986 authority</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">authority</span> <span class="ow">or</span> <span class="s2">"/"</span> <span class="ow">in</span> <span class="n">authority</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="s1">''</span><span class="p">,</span> <span class="mi">400</span>
|
||||||
|
|
||||||
|
<span class="c1"># malformed request / does not have authorisation</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">is_hmac_of</span><span class="p">(</span>
|
||||||
|
<span class="n">CFG</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span>
|
||||||
|
<span class="n">authority</span><span class="o">.</span><span class="n">encode</span><span class="p">(),</span>
|
||||||
|
<span class="n">sxng_request</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'h'</span><span class="p">,</span> <span class="s1">''</span><span class="p">),</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="s1">''</span><span class="p">,</span> <span class="mi">400</span>
|
||||||
|
|
||||||
|
<span class="n">resolver</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">'favicon_resolver'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="c1"># if resolver is empty or not valid, just return HTTP 400.</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resolver</span> <span class="ow">or</span> <span class="n">resolver</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">CFG</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span><span class="p">,</span> <span class="mi">400</span>
|
||||||
|
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">search_favicon</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">Response</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">mimetype</span><span class="o">=</span><span class="n">mime</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Cache-Control'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"max-age=</span><span class="si">{</span><span class="n">CFG</span><span class="o">.</span><span class="n">max_age</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="k">return</span> <span class="n">resp</span>
|
||||||
|
|
||||||
|
<span class="c1"># return default favicon from static path</span>
|
||||||
|
<span class="n">theme</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s2">"theme"</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">fav</span><span class="p">,</span> <span class="n">mimetype</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon</span><span class="p">(</span><span class="n">theme</span><span class="o">=</span><span class="n">theme</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">send_from_directory</span><span class="p">(</span><span class="n">fav</span><span class="o">.</span><span class="n">parent</span><span class="p">,</span> <span class="n">fav</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">mimetype</span><span class="o">=</span><span class="n">mimetype</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="search_favicon">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.search_favicon">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">search_favicon</span><span class="p">(</span><span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Sends the request to the favicon resolver and returns a tuple for the</span>
|
||||||
|
<span class="sd"> favicon. The tuple consists of ``(data, mime)``, if the resolver has not</span>
|
||||||
|
<span class="sd"> determined a favicon, both values are ``None``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``data``:</span>
|
||||||
|
<span class="sd"> Binary data of the favicon.</span>
|
||||||
|
|
||||||
|
<span class="sd"> ``mime``:</span>
|
||||||
|
<span class="sd"> Mime type of the favicon.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">func</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">get_resolver</span><span class="p">(</span><span class="n">resolver</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span>
|
||||||
|
|
||||||
|
<span class="c1"># to avoid superfluous requests to the resolver, first look in the cache</span>
|
||||||
|
<span class="n">data_mime</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">data_mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">data_mime</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">func</span><span class="p">(</span><span class="n">authority</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">CFG</span><span class="o">.</span><span class="n">resolver_timeout</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">except</span> <span class="p">(</span><span class="n">HTTPError</span><span class="p">,</span> <span class="n">SearxEngineResponseException</span><span class="p">):</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
|
||||||
|
<span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">,</span> <span class="n">mime</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="favicon_url">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.favicon_url">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">favicon_url</span><span class="p">(</span><span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Function to generate the image URL used for favicons in SearXNG's result</span>
|
||||||
|
<span class="sd"> lists. The ``authority`` argument (aka netloc / :rfc:`3986`) is usually a</span>
|
||||||
|
<span class="sd"> (sub-) domain name. This function is used in the HTML (jinja) templates.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: html</span>
|
||||||
|
|
||||||
|
<span class="sd"> <div class="favicon"></span>
|
||||||
|
<span class="sd"> <img src="{{ favicon_url(result.parsed_url.netloc) }}"></span>
|
||||||
|
<span class="sd"> </div></span>
|
||||||
|
|
||||||
|
<span class="sd"> The returned URL is a route to :py:obj:`favicon_proxy` REST API.</span>
|
||||||
|
|
||||||
|
<span class="sd"> If the favicon is already in the cache, the returned URL is a `data URL`_</span>
|
||||||
|
<span class="sd"> (something like ``data:image/png;base64,...``). By generating a data url from</span>
|
||||||
|
<span class="sd"> the :py:obj:`.cache.FaviconCache`, additional HTTP roundtripps via the</span>
|
||||||
|
<span class="sd"> :py:obj:`favicon_proxy` are saved. However, it must also be borne in mind</span>
|
||||||
|
<span class="sd"> that data urls are not cached in the client (web browser).</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _data URL: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">resolver</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">'favicon_resolver'</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="c1"># if resolver is empty or not valid, just return nothing.</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">resolver</span> <span class="ow">or</span> <span class="n">resolver</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">CFG</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
||||||
|
<span class="k">return</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="n">data_mime</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">data_mime</span> <span class="o">==</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="c1"># we have already checked, the resolver does not have a favicon</span>
|
||||||
|
<span class="n">theme</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s2">"theme"</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="k">return</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon_data_url</span><span class="p">(</span><span class="n">theme</span><span class="o">=</span><span class="n">theme</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">data_mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">data_mime</span>
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"data:</span><span class="si">{</span><span class="n">mime</span><span class="si">}</span><span class="s2">;base64,</span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">data</span><span class="p">),</span><span class="w"> </span><span class="s1">'utf-8'</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="n">h</span> <span class="o">=</span> <span class="n">new_hmac</span><span class="p">(</span><span class="n">CFG</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span> <span class="n">authority</span><span class="o">.</span><span class="n">encode</span><span class="p">())</span>
|
||||||
|
<span class="n">proxy_url</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">'favicon_proxy'</span><span class="p">)</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlencode</span><span class="p">({</span><span class="s2">"authority"</span><span class="p">:</span> <span class="n">authority</span><span class="p">,</span> <span class="s2">"h"</span><span class="p">:</span> <span class="n">h</span><span class="p">})</span>
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">proxy_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">"</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
215
_modules/searx/favicons/resolvers.html
Normal file
@@ -0,0 +1,215 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.favicons.resolvers — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.favicons.resolvers</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.favicons.resolvers</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Implementations of the favicon *resolvers* that are available in the favicon</span>
|
||||||
|
<span class="sd">proxy by default. A *resolver* is a function that obtains the favicon from an</span>
|
||||||
|
<span class="sd">external source. The *resolver* function receives two arguments (``domain,</span>
|
||||||
|
<span class="sd">timeout``) and returns a tuple ``(data, mime)``.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"DEFAULT_RESOLVER_MAP"</span><span class="p">,</span> <span class="s2">"allesedv"</span><span class="p">,</span> <span class="s2">"duckduckgo"</span><span class="p">,</span> <span class="s2">"google"</span><span class="p">,</span> <span class="s2">"yandex"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">network</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||||
|
|
||||||
|
<span class="n">DEFAULT_RESOLVER_MAP</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Callable</span><span class="p">]</span>
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'favicons.resolvers'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_req_args</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||||
|
<span class="c1"># add the request arguments from the searx.network</span>
|
||||||
|
<span class="n">d</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"raise_for_httperror"</span><span class="p">:</span> <span class="kc">False</span><span class="p">}</span>
|
||||||
|
<span class="n">d</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">d</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="allesedv">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.allesedv">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">allesedv</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Favicon Resolver from allesedv.com / https://favicon.allesedv.com/"""</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://f1.allesedv.com/32/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"fetch favicon from: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># will just return a 200 regardless of the favicon existing or not</span>
|
||||||
|
<span class="c1"># sometimes will be correct size, sometimes not</span>
|
||||||
|
<span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
|
||||||
|
<span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Content-Type'</span><span class="p">]</span>
|
||||||
|
<span class="k">if</span> <span class="n">mime</span> <span class="o">!=</span> <span class="s1">'image/gif'</span><span class="p">:</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="duckduckgo">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.duckduckgo">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">duckduckgo</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Favicon Resolver from duckduckgo.com / https://blog.jim-nielsen.com/2021/displaying-favicons-for-any-domain/"""</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://icons.duckduckgo.com/ip2/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">.ico"</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"fetch favicon from: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># will return a 404 if the favicon does not exist and a 200 if it does,</span>
|
||||||
|
<span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
|
||||||
|
<span class="c1"># api will respond with a 32x32 png image</span>
|
||||||
|
<span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Content-Type'</span><span class="p">]</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="google">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.google">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">google</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Favicon Resolver from google.com"""</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># URL https://www.google.com/s2/favicons?sz=32&domain={domain}" will be</span>
|
||||||
|
<span class="c1"># redirected (HTTP 301 Moved Permanently) to t1.gstatic.com/faviconV2:</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"&url=https://</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">&size=32"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"fetch favicon from: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># will return a 404 if the favicon does not exist and a 200 if it does,</span>
|
||||||
|
<span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span>
|
||||||
|
<span class="c1"># api will respond with a 32x32 png image</span>
|
||||||
|
<span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Content-Type'</span><span class="p">]</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="yandex">
|
||||||
|
<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.yandex">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">yandex</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Favicon Resolver from yandex.com"""</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://favicon.yandex.net/favicon/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"fetch favicon from: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># api will respond with a 16x16 png image, if it doesn't exist, it will be a</span>
|
||||||
|
<span class="c1"># 1x1 png image (70 bytes)</span>
|
||||||
|
<span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">></span> <span class="mi">70</span><span class="p">:</span>
|
||||||
|
<span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Content-Type'</span><span class="p">]</span>
|
||||||
|
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span>
|
||||||
|
<span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">DEFAULT_RESOLVER_MAP</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"allesedv"</span><span class="p">:</span> <span class="n">allesedv</span><span class="p">,</span>
|
||||||
|
<span class="s2">"duckduckgo"</span><span class="p">:</span> <span class="n">duckduckgo</span><span class="p">,</span>
|
||||||
|
<span class="s2">"google"</span><span class="p">:</span> <span class="n">google</span><span class="p">,</span>
|
||||||
|
<span class="s2">"yandex"</span><span class="p">:</span> <span class="n">yandex</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
326
_modules/searx/infopage.html
Normal file
@@ -0,0 +1,326 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.infopage — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.infopage</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.infopage</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Render SearXNG instance documentation.</span>
|
||||||
|
|
||||||
|
<span class="sd">Usage in a Flask app route:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> from searx import infopage</span>
|
||||||
|
<span class="sd"> from searx.extended_types import sxng_request</span>
|
||||||
|
|
||||||
|
<span class="sd"> _INFO_PAGES = infopage.InfoPageSet(infopage.MistletoePage)</span>
|
||||||
|
|
||||||
|
<span class="sd"> @app.route('/info/<pagename>', methods=['GET'])</span>
|
||||||
|
<span class="sd"> def info(pagename):</span>
|
||||||
|
|
||||||
|
<span class="sd"> locale = sxng_request.preferences.get_value('locale')</span>
|
||||||
|
<span class="sd"> page = _INFO_PAGES.get_page(pagename, locale)</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'InfoPage'</span><span class="p">,</span> <span class="s1">'InfoPageSet'</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">os.path</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib.parse</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">functools</span><span class="w"> </span><span class="kn">import</span> <span class="n">cached_property</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">jinja2</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">url_for</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">markdown_it</span><span class="w"> </span><span class="kn">import</span> <span class="n">MarkdownIt</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">..</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_setting</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">..version</span><span class="w"> </span><span class="kn">import</span> <span class="n">GIT_URL</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">..locales</span><span class="w"> </span><span class="kn">import</span> <span class="n">LOCALE_NAMES</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s1">'searx.infopage'</span><span class="p">)</span>
|
||||||
|
<span class="n">_INFO_FOLDER</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span>
|
||||||
|
<span class="n">INFO_PAGES</span><span class="p">:</span> <span class="s1">'InfoPageSet'</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__getattr__</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">name</span> <span class="o">==</span> <span class="s1">'INFO_PAGES'</span><span class="p">:</span>
|
||||||
|
<span class="k">global</span> <span class="n">INFO_PAGES</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
<span class="n">INFO_PAGES</span> <span class="o">=</span> <span class="n">InfoPageSet</span><span class="p">()</span>
|
||||||
|
<span class="k">return</span> <span class="n">INFO_PAGES</span>
|
||||||
|
|
||||||
|
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"module </span><span class="si">{</span><span class="vm">__name__</span><span class="si">!r}</span><span class="s2"> has no attribute </span><span class="si">{</span><span class="n">name</span><span class="si">!r}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPage">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">InfoPage</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A page of the :py:obj:`online documentation <InfoPageSet>`."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fname</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">fname</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">fname</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPage.raw_content">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.raw_content">[docs]</a>
|
||||||
|
<span class="nd">@cached_property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">raw_content</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Raw content of the page (without any jinja rendering)"""</span>
|
||||||
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fname</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPage.content">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.content">[docs]</a>
|
||||||
|
<span class="nd">@cached_property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">content</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Content of the page (rendered in a Jinja context)"""</span>
|
||||||
|
<span class="n">ctx</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_ctx</span><span class="p">()</span>
|
||||||
|
<span class="n">template</span> <span class="o">=</span> <span class="n">jinja2</span><span class="o">.</span><span class="n">Environment</span><span class="p">()</span><span class="o">.</span><span class="n">from_string</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">raw_content</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">template</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="o">**</span><span class="n">ctx</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPage.title">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.title">[docs]</a>
|
||||||
|
<span class="nd">@cached_property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">title</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Title of the content (without any markup)"""</span>
|
||||||
|
<span class="n">_t</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">raw_content</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">l</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'# '</span><span class="p">):</span>
|
||||||
|
<span class="n">_t</span> <span class="o">=</span> <span class="n">l</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">'# '</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">_t</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPage.html">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.html">[docs]</a>
|
||||||
|
<span class="nd">@cached_property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">html</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Render Markdown (CommonMark_) to HTML by using markdown-it-py_.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _CommonMark: https://commonmark.org/</span>
|
||||||
|
<span class="sd"> .. _markdown-it-py: https://github.com/executablebooks/markdown-it-py</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="p">(</span>
|
||||||
|
<span class="n">MarkdownIt</span><span class="p">(</span><span class="s2">"commonmark"</span><span class="p">,</span> <span class="p">{</span><span class="s2">"typographer"</span><span class="p">:</span> <span class="kc">True</span><span class="p">})</span><span class="o">.</span><span class="n">enable</span><span class="p">([</span><span class="s2">"replacements"</span><span class="p">,</span> <span class="s2">"smartquotes"</span><span class="p">])</span><span class="o">.</span><span class="n">render</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">content</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPage.get_ctx">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPage.get_ctx">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_ctx</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Jinja context to render :py:obj:`InfoPage.content`"""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_md_link</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">url</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="n">url_for</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">_external</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"[</span><span class="si">%s</span><span class="s2">](</span><span class="si">%s</span><span class="s2">)"</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_md_search</span><span class="p">(</span><span class="n">query</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">?q=</span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="p">(</span><span class="n">url_for</span><span class="p">(</span><span class="s1">'search'</span><span class="p">,</span> <span class="n">_external</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">query</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="s1">'[</span><span class="si">%s</span><span class="s1">](</span><span class="si">%s</span><span class="s1">)'</span> <span class="o">%</span> <span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">ctx</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">ctx</span><span class="p">[</span><span class="s1">'GIT_URL'</span><span class="p">]</span> <span class="o">=</span> <span class="n">GIT_URL</span>
|
||||||
|
<span class="n">ctx</span><span class="p">[</span><span class="s1">'get_setting'</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_setting</span>
|
||||||
|
<span class="n">ctx</span><span class="p">[</span><span class="s1">'link'</span><span class="p">]</span> <span class="o">=</span> <span class="n">_md_link</span>
|
||||||
|
<span class="n">ctx</span><span class="p">[</span><span class="s1">'search'</span><span class="p">]</span> <span class="o">=</span> <span class="n">_md_search</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">ctx</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s1">'<</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s1"> fname=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">fname</span><span class="si">!r}</span><span class="s1">>'</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPageSet">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">InfoPageSet</span><span class="p">:</span> <span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Cached rendering of the online documentation a SearXNG instance has.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param page_class: render online documentation by :py:obj:`InfoPage` parser.</span>
|
||||||
|
<span class="sd"> :type page_class: :py:obj:`InfoPage`</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param info_folder: information directory</span>
|
||||||
|
<span class="sd"> :type info_folder: str</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">page_class</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">InfoPage</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">info_folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">page_class</span><span class="p">:</span> <span class="nb">type</span><span class="p">[</span><span class="n">InfoPage</span><span class="p">]</span> <span class="o">=</span> <span class="n">page_class</span> <span class="ow">or</span> <span class="n">InfoPage</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">folder</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">info_folder</span> <span class="ow">or</span> <span class="n">_INFO_FOLDER</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""location of the Markdown files"""</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">InfoPage</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""default language"""</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">locales</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span> <span class="k">for</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">_INFO_FOLDER</span><span class="p">)</span> <span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">'-'</span><span class="p">)</span> <span class="ow">in</span> <span class="n">LOCALE_NAMES</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""list of supported languages (aka locales)"""</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="s1">'search-syntax'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'about'</span><span class="p">,</span>
|
||||||
|
<span class="s1">'donate'</span><span class="p">,</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""list of articles in the online documentation"""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPageSet.get_page">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet.get_page">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_page</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pagename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">locale</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Return ``pagename`` instance of :py:obj:`InfoPage`</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param pagename: name of the page, a value from :py:obj:`InfoPageSet.toc`</span>
|
||||||
|
<span class="sd"> :type pagename: str</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param locale: language of the page, e.g. ``en``, ``zh_Hans_CN``</span>
|
||||||
|
<span class="sd"> (default: :py:obj:`InfoPageSet.i18n_origin`)</span>
|
||||||
|
<span class="sd"> :type locale: str</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">pagename</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">locales</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">cache_key</span> <span class="o">=</span> <span class="p">(</span><span class="n">pagename</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">cache_key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="c1"># not yet instantiated</span>
|
||||||
|
|
||||||
|
<span class="n">fname</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">folder</span><span class="p">,</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">),</span> <span class="n">pagename</span><span class="p">)</span> <span class="o">+</span> <span class="s1">'.md'</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">fname</span><span class="p">):</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'file </span><span class="si">%s</span><span class="s1"> does not exists'</span><span class="p">,</span> <span class="n">fname</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">page_class</span><span class="p">(</span><span class="n">fname</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">CACHE</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">page</span>
|
||||||
|
<span class="k">return</span> <span class="n">page</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="InfoPageSet.iter_pages">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.infopage.html#searx.infopage.InfoPageSet.iter_pages">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">iter_pages</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">locale</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">fallback_to_default</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Iterate over all pages of the TOC"""</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
|
||||||
|
<span class="k">for</span> <span class="n">page_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">toc</span><span class="p">:</span>
|
||||||
|
<span class="n">page_locale</span> <span class="o">=</span> <span class="n">locale</span>
|
||||||
|
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_page</span><span class="p">(</span><span class="n">page_name</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">fallback_to_default</span> <span class="ow">and</span> <span class="n">page</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">page_locale</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span>
|
||||||
|
<span class="n">page</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_page</span><span class="p">(</span><span class="n">page_name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">locale_default</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">page</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="c1"># page is None if the page was deleted by the administrator</span>
|
||||||
|
<span class="k">yield</span> <span class="n">page_name</span><span class="p">,</span> <span class="n">page_locale</span><span class="p">,</span> <span class="n">page</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
367
_modules/searx/limiter.html
Normal file
@@ -0,0 +1,367 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.limiter — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.limiter</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.limiter</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Bot protection / IP rate limitation. The intention of rate limitation is to</span>
|
||||||
|
<span class="sd">limit suspicious requests from an IP. The motivation behind this is the fact</span>
|
||||||
|
<span class="sd">that SearXNG passes through requests from bots and is thus classified as a bot</span>
|
||||||
|
<span class="sd">itself. As a result, the SearXNG engine then receives a CAPTCHA or is blocked</span>
|
||||||
|
<span class="sd">by the search engine (the origin) in some other way.</span>
|
||||||
|
|
||||||
|
<span class="sd">To avoid blocking, the requests from bots to SearXNG must also be blocked, this</span>
|
||||||
|
<span class="sd">is the task of the limiter. To perform this task, the limiter uses the methods</span>
|
||||||
|
<span class="sd">from the :ref:`botdetection`:</span>
|
||||||
|
|
||||||
|
<span class="sd">- Analysis of the HTTP header in the request / :ref:`botdetection probe headers`</span>
|
||||||
|
<span class="sd"> can be easily bypassed.</span>
|
||||||
|
|
||||||
|
<span class="sd">- Block and pass lists in which IPs are listed / :ref:`botdetection ip_lists`</span>
|
||||||
|
<span class="sd"> are hard to maintain, since the IPs of bots are not all known and change over</span>
|
||||||
|
<span class="sd"> the time.</span>
|
||||||
|
|
||||||
|
<span class="sd">- Detection & dynamically :ref:`botdetection rate limit` of bots based on the</span>
|
||||||
|
<span class="sd"> behavior of the requests. For dynamically changeable IP lists a Valkey</span>
|
||||||
|
<span class="sd"> database is needed.</span>
|
||||||
|
|
||||||
|
<span class="sd">The prerequisite for IP based methods is the correct determination of the IP of</span>
|
||||||
|
<span class="sd">the client. The IP of the client is determined via the X-Forwarded-For_ HTTP</span>
|
||||||
|
<span class="sd">header.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. attention::</span>
|
||||||
|
|
||||||
|
<span class="sd"> A correct setup of the HTTP request headers ``X-Forwarded-For`` and</span>
|
||||||
|
<span class="sd"> ``X-Real-IP`` is essential to be able to assign a request to an IP correctly:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - `NGINX RequestHeader`_</span>
|
||||||
|
<span class="sd"> - `Apache RequestHeader`_</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _X-Forwarded-For:</span>
|
||||||
|
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For</span>
|
||||||
|
<span class="sd">.. _NGINX RequestHeader:</span>
|
||||||
|
<span class="sd"> https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site</span>
|
||||||
|
<span class="sd">.. _Apache RequestHeader:</span>
|
||||||
|
<span class="sd"> https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site</span>
|
||||||
|
|
||||||
|
<span class="sd">Enable Limiter</span>
|
||||||
|
<span class="sd">==============</span>
|
||||||
|
|
||||||
|
<span class="sd">To enable the limiter activate:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> server:</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> limiter: true # rate limit the number of request on the instance, block some bots</span>
|
||||||
|
|
||||||
|
<span class="sd">and set the valkey-url connection. Check the value, it depends on your valkey DB</span>
|
||||||
|
<span class="sd">(see :ref:`settings valkey`), by example:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> valkey:</span>
|
||||||
|
<span class="sd"> url: valkey://localhost:6379/0</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">Configure Limiter</span>
|
||||||
|
<span class="sd">=================</span>
|
||||||
|
|
||||||
|
<span class="sd">The methods of :ref:`botdetection` the limiter uses are configured in a local</span>
|
||||||
|
<span class="sd">file ``/etc/searxng/limiter.toml``. The defaults are shown in limiter.toml_ /</span>
|
||||||
|
<span class="sd">Don't copy all values to your local configuration, just enable what you need by</span>
|
||||||
|
<span class="sd">overwriting the defaults. For instance to activate the ``link_token`` method in</span>
|
||||||
|
<span class="sd">the :ref:`botdetection.ip_limit` you only need to set this option to ``true``:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: toml</span>
|
||||||
|
|
||||||
|
<span class="sd"> [botdetection.ip_limit]</span>
|
||||||
|
<span class="sd"> link_token = true</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _limiter.toml:</span>
|
||||||
|
|
||||||
|
<span class="sd">``limiter.toml``</span>
|
||||||
|
<span class="sd">================</span>
|
||||||
|
|
||||||
|
<span class="sd">In this file the limiter finds the configuration of the :ref:`botdetection`:</span>
|
||||||
|
|
||||||
|
<span class="sd">- :ref:`botdetection ip_lists`</span>
|
||||||
|
<span class="sd">- :ref:`botdetection rate limit`</span>
|
||||||
|
<span class="sd">- :ref:`botdetection probe headers`</span>
|
||||||
|
|
||||||
|
<span class="sd">.. kernel-include:: $SOURCEDIR/limiter.toml</span>
|
||||||
|
<span class="sd"> :code: toml</span>
|
||||||
|
|
||||||
|
<span class="sd">Implementation</span>
|
||||||
|
<span class="sd">==============</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="n">ip_address</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">sys</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">werkzeug</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">searx.compat</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">logger</span><span class="p">,</span>
|
||||||
|
<span class="n">valkeydb</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">botdetection</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">sxng_request</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.botdetection</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">config</span><span class="p">,</span>
|
||||||
|
<span class="n">http_accept</span><span class="p">,</span>
|
||||||
|
<span class="n">http_accept_encoding</span><span class="p">,</span>
|
||||||
|
<span class="n">http_accept_language</span><span class="p">,</span>
|
||||||
|
<span class="n">http_user_agent</span><span class="p">,</span>
|
||||||
|
<span class="n">http_sec_fetch</span><span class="p">,</span>
|
||||||
|
<span class="n">ip_limit</span><span class="p">,</span>
|
||||||
|
<span class="n">ip_lists</span><span class="p">,</span>
|
||||||
|
<span class="n">get_network</span><span class="p">,</span>
|
||||||
|
<span class="n">dump_request</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># the configuration are limiter.toml and "limiter" in settings.yml so, for</span>
|
||||||
|
<span class="c1"># coherency, the logger is "limiter"</span>
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'limiter'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">CFG</span><span class="p">:</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">_INSTALLED</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">LIMITER_CFG_SCHEMA</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span class="n">parent</span> <span class="o">/</span> <span class="s2">"limiter.toml"</span>
|
||||||
|
<span class="sd">"""Base configuration (schema) of the botdetection."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_cfg">
|
||||||
|
<a class="viewcode-back" href="../../admin/searx.limiter.html#searx.limiter.get_cfg">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_cfg</span><span class="p">()</span> <span class="o">-></span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns SearXNG's global limiter configuration."""</span>
|
||||||
|
<span class="k">global</span> <span class="n">CFG</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">CFG</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">settings_loader</span> <span class="c1"># pylint: disable=import-outside-toplevel</span>
|
||||||
|
|
||||||
|
<span class="n">cfg_file</span> <span class="o">=</span> <span class="p">(</span><span class="n">settings_loader</span><span class="o">.</span><span class="n">get_user_cfg_folder</span><span class="p">()</span> <span class="ow">or</span> <span class="n">Path</span><span class="p">(</span><span class="s2">"/etc/searxng"</span><span class="p">))</span> <span class="o">/</span> <span class="s2">"limiter.toml"</span>
|
||||||
|
<span class="n">CFG</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">Config</span><span class="o">.</span><span class="n">from_toml</span><span class="p">(</span><span class="n">LIMITER_CFG_SCHEMA</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">,</span> <span class="n">searx</span><span class="o">.</span><span class="n">compat</span><span class="o">.</span><span class="n">LIMITER_CFG_DEPRECATED</span><span class="p">)</span>
|
||||||
|
<span class="n">searx</span><span class="o">.</span><span class="n">compat</span><span class="o">.</span><span class="n">limiter_fix_cfg</span><span class="p">(</span><span class="n">CFG</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">CFG</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">filter_request</span><span class="p">(</span><span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">)</span> <span class="o">-></span> <span class="n">werkzeug</span><span class="o">.</span><span class="n">Response</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-return-statements</span>
|
||||||
|
|
||||||
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">get_cfg</span><span class="p">()</span>
|
||||||
|
<span class="n">real_ip</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">)</span>
|
||||||
|
<span class="n">network</span> <span class="o">=</span> <span class="n">get_network</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">path</span> <span class="o">==</span> <span class="s1">'/healthz'</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># link-local</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">network</span><span class="o">.</span><span class="n">is_link_local</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># block- & pass- lists</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># 1. The IP of the request is first checked against the pass-list; if the IP</span>
|
||||||
|
<span class="c1"># matches an entry in the list, the request is not blocked.</span>
|
||||||
|
<span class="c1"># 2. If no matching entry is found in the pass-list, then a check is made against</span>
|
||||||
|
<span class="c1"># the block list; if the IP matches an entry in the list, the request is</span>
|
||||||
|
<span class="c1"># blocked.</span>
|
||||||
|
<span class="c1"># 3. If the IP is not in either list, the request is not blocked.</span>
|
||||||
|
|
||||||
|
<span class="n">match</span><span class="p">,</span> <span class="n">msg</span> <span class="o">=</span> <span class="n">ip_lists</span><span class="o">.</span><span class="n">pass_ip</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">match</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"PASS </span><span class="si">%s</span><span class="s2">: matched PASSLIST - </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">match</span><span class="p">,</span> <span class="n">msg</span> <span class="o">=</span> <span class="n">ip_lists</span><span class="o">.</span><span class="n">block_ip</span><span class="p">(</span><span class="n">real_ip</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">match</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"BLOCK </span><span class="si">%s</span><span class="s2">: matched BLOCKLIST - </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">network</span><span class="o">.</span><span class="n">compressed</span><span class="p">,</span> <span class="n">msg</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">make_response</span><span class="p">((</span><span class="s1">'IP is on BLOCKLIST - </span><span class="si">%s</span><span class="s1">'</span> <span class="o">%</span> <span class="n">msg</span><span class="p">,</span> <span class="mi">429</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># methods applied on all requests</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">func</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="n">http_user_agent</span><span class="p">,</span>
|
||||||
|
<span class="p">]:</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">func</span><span class="o">.</span><span class="n">filter_request</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"NOT OK (</span><span class="si">{</span><span class="n">func</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">network</span><span class="si">}</span><span class="s2">: %s"</span><span class="p">,</span> <span class="n">dump_request</span><span class="p">(</span><span class="n">sxng_request</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">val</span>
|
||||||
|
|
||||||
|
<span class="c1"># methods applied on /search requests</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">path</span> <span class="o">==</span> <span class="s1">'/search'</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">func</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="n">http_accept</span><span class="p">,</span>
|
||||||
|
<span class="n">http_accept_encoding</span><span class="p">,</span>
|
||||||
|
<span class="n">http_accept_language</span><span class="p">,</span>
|
||||||
|
<span class="n">http_user_agent</span><span class="p">,</span>
|
||||||
|
<span class="n">http_sec_fetch</span><span class="p">,</span>
|
||||||
|
<span class="n">ip_limit</span><span class="p">,</span>
|
||||||
|
<span class="p">]:</span>
|
||||||
|
<span class="n">val</span> <span class="o">=</span> <span class="n">func</span><span class="o">.</span><span class="n">filter_request</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">cfg</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"NOT OK (</span><span class="si">{</span><span class="n">func</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">network</span><span class="si">}</span><span class="s2">: %s"</span><span class="p">,</span> <span class="n">dump_request</span><span class="p">(</span><span class="n">sxng_request</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">val</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"OK </span><span class="si">{</span><span class="n">network</span><span class="si">}</span><span class="s2">: %s"</span><span class="p">,</span> <span class="n">dump_request</span><span class="p">(</span><span class="n">sxng_request</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="pre_request">
|
||||||
|
<a class="viewcode-back" href="../../admin/searx.limiter.html#searx.limiter.pre_request">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">pre_request</span><span class="p">():</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""See :py:obj:`flask.Flask.before_request`"""</span>
|
||||||
|
<span class="k">return</span> <span class="n">filter_request</span><span class="p">(</span><span class="n">sxng_request</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="is_installed">
|
||||||
|
<a class="viewcode-back" href="../../admin/searx.limiter.html#searx.limiter.is_installed">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_installed</span><span class="p">():</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns ``True`` if limiter is active and a valkey DB is available."""</span>
|
||||||
|
<span class="k">return</span> <span class="n">_INSTALLED</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="initialize">
|
||||||
|
<a class="viewcode-back" href="../../admin/searx.limiter.html#searx.limiter.initialize">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">initialize</span><span class="p">(</span><span class="n">app</span><span class="p">:</span> <span class="n">flask</span><span class="o">.</span><span class="n">Flask</span><span class="p">,</span> <span class="n">settings</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Install the limiter"""</span>
|
||||||
|
<span class="k">global</span> <span class="n">_INSTALLED</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
|
||||||
|
<span class="c1"># even if the limiter is not activated, the botdetection must be activated</span>
|
||||||
|
<span class="c1"># (e.g. the self_info plugin uses the botdetection to get client IP)</span>
|
||||||
|
|
||||||
|
<span class="n">cfg</span> <span class="o">=</span> <span class="n">get_cfg</span><span class="p">()</span>
|
||||||
|
<span class="n">valkey_client</span> <span class="o">=</span> <span class="n">valkeydb</span><span class="o">.</span><span class="n">client</span><span class="p">()</span>
|
||||||
|
<span class="n">botdetection</span><span class="o">.</span><span class="n">init</span><span class="p">(</span><span class="n">cfg</span><span class="p">,</span> <span class="n">valkey_client</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">settings</span><span class="p">[</span><span class="s1">'server'</span><span class="p">][</span><span class="s1">'limiter'</span><span class="p">]</span> <span class="ow">or</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'server'</span><span class="p">][</span><span class="s1">'public_instance'</span><span class="p">]):</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">valkey_client</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
|
||||||
|
<span class="s2">"The limiter requires Valkey, please consult the documentation: "</span>
|
||||||
|
<span class="s2">"https://docs.searxng.org/admin/searx.limiter.html"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'server'</span><span class="p">][</span><span class="s1">'public_instance'</span><span class="p">]:</span>
|
||||||
|
<span class="n">sys</span><span class="o">.</span><span class="n">exit</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
|
||||||
|
<span class="n">_INSTALLED</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">settings</span><span class="p">[</span><span class="s1">'server'</span><span class="p">][</span><span class="s1">'public_instance'</span><span class="p">]:</span>
|
||||||
|
<span class="c1"># overwrite limiter.toml setting</span>
|
||||||
|
<span class="n">cfg</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s1">'botdetection.ip_limit.link_token'</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">app</span><span class="o">.</span><span class="n">before_request</span><span class="p">(</span><span class="n">pre_request</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
598
_modules/searx/locales.html
Normal file
@@ -0,0 +1,598 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.locales — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.locales</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.locales</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">SearXNG’s locale data</span>
|
||||||
|
<span class="sd">=====================</span>
|
||||||
|
|
||||||
|
<span class="sd">The variables :py:obj:`RTL_LOCALES` and :py:obj:`LOCALE_NAMES` are loaded from</span>
|
||||||
|
<span class="sd">:origin:`searx/data/locales.json` / see :py:obj:`locales_initialize` and</span>
|
||||||
|
<span class="sd">:ref:`update_locales.py`.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> Whenever the value of :py:obj:`ADDITIONAL_TRANSLATIONS` or</span>
|
||||||
|
<span class="sd"> :py:obj:`LOCALE_BEST_MATCH` is modified, the</span>
|
||||||
|
<span class="sd"> :origin:`searx/data/locales.json` needs to be rebuild::</span>
|
||||||
|
|
||||||
|
<span class="sd"> ./manage data.locales</span>
|
||||||
|
|
||||||
|
<span class="sd">SearXNG's locale codes</span>
|
||||||
|
<span class="sd">======================</span>
|
||||||
|
|
||||||
|
<span class="sd">.. automodule:: searx.sxng_locales</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="sd">SearXNG’s locale implementations</span>
|
||||||
|
<span class="sd">================================</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">babel.support</span><span class="w"> </span><span class="kn">import</span> <span class="n">Translations</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.languages</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.core</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask_babel</span> <span class="c1"># pyright: ignore[reportMissingTypeStubs]</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask.ctx</span><span class="w"> </span><span class="kn">import</span> <span class="n">has_request_context</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">data</span><span class="p">,</span>
|
||||||
|
<span class="n">logger</span><span class="p">,</span>
|
||||||
|
<span class="n">searx_dir</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">sxng_request</span>
|
||||||
|
|
||||||
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'locales'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># safe before monkey patching flask_babel.get_translations</span>
|
||||||
|
<span class="n">_flask_babel_get_translations</span> <span class="o">=</span> <span class="n">flask_babel</span><span class="o">.</span><span class="n">get_translations</span>
|
||||||
|
|
||||||
|
<span class="n">LOCALE_NAMES</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="sd">"""Mapping of locales and their description. Locales e.g. 'fr' or 'pt-BR' (see</span>
|
||||||
|
<span class="sd">:py:obj:`locales_initialize`).</span>
|
||||||
|
|
||||||
|
<span class="sd">:meta hide-value:</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">RTL_LOCALES</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="sd">"""List of *Right-To-Left* locales e.g. 'he' or 'fa-IR' (see</span>
|
||||||
|
<span class="sd">:py:obj:`locales_initialize`)."""</span>
|
||||||
|
|
||||||
|
<span class="n">ADDITIONAL_TRANSLATIONS</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"dv"</span><span class="p">:</span> <span class="s2">"ދިވެހި (Dhivehi)"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"oc"</span><span class="p">:</span> <span class="s2">"Occitan"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"szl"</span><span class="p">:</span> <span class="s2">"Ślōnski (Silesian)"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"pap"</span><span class="p">:</span> <span class="s2">"Papiamento"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""Additional languages SearXNG has translations for but not supported by</span>
|
||||||
|
<span class="sd">python-babel (see :py:obj:`locales_initialize`)."""</span>
|
||||||
|
|
||||||
|
<span class="n">LOCALE_BEST_MATCH</span> <span class="o">=</span> <span class="p">{</span>
|
||||||
|
<span class="s2">"dv"</span><span class="p">:</span> <span class="s2">"si"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"oc"</span><span class="p">:</span> <span class="s1">'fr-FR'</span><span class="p">,</span>
|
||||||
|
<span class="s2">"szl"</span><span class="p">:</span> <span class="s2">"pl"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"nl-BE"</span><span class="p">:</span> <span class="s2">"nl"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"zh-HK"</span><span class="p">:</span> <span class="s2">"zh-Hant-TW"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"pap"</span><span class="p">:</span> <span class="s2">"pt-BR"</span><span class="p">,</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
<span class="sd">"""Map a locale we do not have a translations for to a locale we have a</span>
|
||||||
|
<span class="sd">translation for. By example: use Taiwan version of the translation for Hong</span>
|
||||||
|
<span class="sd">Kong."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">localeselector</span><span class="p">():</span>
|
||||||
|
<span class="n">locale</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||||
|
<span class="k">if</span> <span class="n">has_request_context</span><span class="p">():</span>
|
||||||
|
<span class="n">value</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">'locale'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">value</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">value</span>
|
||||||
|
|
||||||
|
<span class="c1"># first, set the language that is not supported by babel</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_request</span><span class="o">.</span><span class="n">form</span><span class="p">[</span><span class="s1">'use-translation'</span><span class="p">]</span> <span class="o">=</span> <span class="n">locale</span>
|
||||||
|
|
||||||
|
<span class="c1"># second, map locale to a value python-babel supports</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">LOCALE_BEST_MATCH</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">locale</span><span class="p">,</span> <span class="n">locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">locale</span> <span class="o">==</span> <span class="s1">''</span><span class="p">:</span>
|
||||||
|
<span class="c1"># if there is an error loading the preferences</span>
|
||||||
|
<span class="c1"># the locale is going to be ''</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="s1">'en'</span>
|
||||||
|
|
||||||
|
<span class="c1"># babel uses underscore instead of hyphen.</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'-'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">locale</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_translations">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_translations">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_translations</span><span class="p">():</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Monkey patch of :py:obj:`flask_babel.get_translations`"""</span>
|
||||||
|
<span class="k">if</span> <span class="n">has_request_context</span><span class="p">():</span>
|
||||||
|
<span class="n">use_translation</span> <span class="o">=</span> <span class="n">sxng_request</span><span class="o">.</span><span class="n">form</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'use-translation'</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">use_translation</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="p">:</span>
|
||||||
|
<span class="n">babel_ext</span> <span class="o">=</span> <span class="n">flask_babel</span><span class="o">.</span><span class="n">current_app</span><span class="o">.</span><span class="n">extensions</span><span class="p">[</span><span class="s1">'babel'</span><span class="p">]</span>
|
||||||
|
<span class="k">return</span> <span class="n">Translations</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">babel_ext</span><span class="o">.</span><span class="n">translation_directories</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">use_translation</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">_flask_babel_get_translations</span><span class="p">()</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">_TR_LOCALES</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_translation_locales">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_translation_locales">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_translation_locales</span><span class="p">()</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns the list of translation locales (*underscore*). The list is</span>
|
||||||
|
<span class="sd"> generated from the translation folders in :origin:`searx/translations`"""</span>
|
||||||
|
|
||||||
|
<span class="k">global</span> <span class="n">_TR_LOCALES</span> <span class="c1"># pylint:disable=global-statement</span>
|
||||||
|
<span class="k">if</span> <span class="n">_TR_LOCALES</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">_TR_LOCALES</span>
|
||||||
|
|
||||||
|
<span class="n">tr_locales</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">folder</span> <span class="ow">in</span> <span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">searx_dir</span><span class="p">)</span> <span class="o">/</span> <span class="s1">'translations'</span><span class="p">)</span><span class="o">.</span><span class="n">iterdir</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">folder</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">folder</span> <span class="o">/</span> <span class="s1">'LC_MESSAGES'</span><span class="p">)</span><span class="o">.</span><span class="n">is_dir</span><span class="p">():</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">tr_locales</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">folder</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
|
<span class="n">_TR_LOCALES</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">tr_locales</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">_TR_LOCALES</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="locales_initialize">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.locales_initialize">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">locales_initialize</span><span class="p">():</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialize locales environment of the SearXNG session.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - monkey patch :py:obj:`flask_babel.get_translations` by :py:obj:`get_translations`</span>
|
||||||
|
<span class="sd"> - init global names :py:obj:`LOCALE_NAMES`, :py:obj:`RTL_LOCALES`</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">flask_babel</span><span class="o">.</span><span class="n">get_translations</span> <span class="o">=</span> <span class="n">get_translations</span>
|
||||||
|
<span class="n">LOCALE_NAMES</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">LOCALES</span><span class="p">[</span><span class="s2">"LOCALE_NAMES"</span><span class="p">])</span>
|
||||||
|
<span class="n">RTL_LOCALES</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">LOCALES</span><span class="p">[</span><span class="s2">"RTL_LOCALES"</span><span class="p">])</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="region_tag">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.region_tag">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">region_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">:</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns SearXNG's region tag from the locale (e.g. zh-TW , en-US)."""</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">'babel.Locale </span><span class="si">%s</span><span class="s1">: missed a territory'</span> <span class="o">%</span> <span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="language_tag">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.language_tag">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">:</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns SearXNG's language tag from the locale and if exits, the tag</span>
|
||||||
|
<span class="sd"> includes the script name (e.g. en, zh_Hant).</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">sxng_lang</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">script</span><span class="p">:</span>
|
||||||
|
<span class="n">sxng_lang</span> <span class="o">+=</span> <span class="s1">'_'</span> <span class="o">+</span> <span class="n">locale</span><span class="o">.</span><span class="n">script</span>
|
||||||
|
<span class="k">return</span> <span class="n">sxng_lang</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_locale">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_locale">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_locale</span><span class="p">(</span><span class="n">locale_tag</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a :py:obj:`babel.Locale` object parsed from argument</span>
|
||||||
|
<span class="sd"> ``locale_tag``"""</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">locale_tag</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">locale</span>
|
||||||
|
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_official_locales">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_official_locales">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_official_locales</span><span class="p">(</span>
|
||||||
|
<span class="n">territory</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">languages</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">regional</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">de_facto</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="nb">set</span><span class="p">[</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a list of :py:obj:`babel.Locale` with languages from</span>
|
||||||
|
<span class="sd"> :py:obj:`babel.languages.get_official_languages`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param territory: The territory (country or region) code.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param languages: A list of language codes the languages from</span>
|
||||||
|
<span class="sd"> :py:obj:`babel.languages.get_official_languages` should be in</span>
|
||||||
|
<span class="sd"> (intersection). If this argument is ``None``, all official languages in</span>
|
||||||
|
<span class="sd"> this territory are used.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param regional: If the regional flag is set, then languages which are</span>
|
||||||
|
<span class="sd"> regionally official are also returned.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param de_facto: If the de_facto flag is set to `False`, then languages</span>
|
||||||
|
<span class="sd"> which are “de facto” official are not returned.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">ret_val</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="n">o_languages</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">territory</span><span class="p">,</span> <span class="n">regional</span><span class="o">=</span><span class="n">regional</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="n">de_facto</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">languages</span><span class="p">:</span>
|
||||||
|
<span class="n">languages</span> <span class="o">=</span> <span class="p">[</span><span class="n">l</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">]</span>
|
||||||
|
<span class="n">o_languages</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">l</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">o_languages</span> <span class="k">if</span> <span class="n">l</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">in</span> <span class="n">languages</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">lang</span> <span class="ow">in</span> <span class="n">o_languages</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">lang</span> <span class="o">+</span> <span class="s1">'_'</span> <span class="o">+</span> <span class="n">territory</span><span class="p">)</span>
|
||||||
|
<span class="n">ret_val</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">ret_val</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="get_engine_locale">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.get_engine_locale">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">engine_locales</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">default</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Return engine's language (aka locale) string that best fits to argument</span>
|
||||||
|
<span class="sd"> ``searxng_locale``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to</span>
|
||||||
|
<span class="sd"> corresponding *engine locales*::</span>
|
||||||
|
|
||||||
|
<span class="sd"> <engine>: {</span>
|
||||||
|
<span class="sd"> # SearXNG string : engine-string</span>
|
||||||
|
<span class="sd"> 'ca-ES' : 'ca_ES',</span>
|
||||||
|
<span class="sd"> 'fr-BE' : 'fr_BE',</span>
|
||||||
|
<span class="sd"> 'fr-CA' : 'fr_CA',</span>
|
||||||
|
<span class="sd"> 'fr-CH' : 'fr_CH',</span>
|
||||||
|
<span class="sd"> 'fr' : 'fr_FR',</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> 'pl-PL' : 'pl_PL',</span>
|
||||||
|
<span class="sd"> 'pt-PT' : 'pt_PT'</span>
|
||||||
|
<span class="sd"> ..</span>
|
||||||
|
<span class="sd"> 'zh' : 'zh'</span>
|
||||||
|
<span class="sd"> 'zh_Hans' : 'zh'</span>
|
||||||
|
<span class="sd"> 'zh_Hant' : 'zh_TW'</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> The *SearXNG locale* string has to be known by babel!</span>
|
||||||
|
|
||||||
|
<span class="sd"> If there is no direct 1:1 mapping, this functions tries to narrow down</span>
|
||||||
|
<span class="sd"> engine's language (locale). If no value can be determined by these</span>
|
||||||
|
<span class="sd"> approximation attempts the ``default`` value is returned.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Assumptions:</span>
|
||||||
|
|
||||||
|
<span class="sd"> A. When user select a language the results should be optimized according to</span>
|
||||||
|
<span class="sd"> the selected language.</span>
|
||||||
|
|
||||||
|
<span class="sd"> B. When user select a language and a territory the results should be</span>
|
||||||
|
<span class="sd"> optimized with first priority on territory and second on language.</span>
|
||||||
|
|
||||||
|
<span class="sd"> First approximation rule (*by territory*):</span>
|
||||||
|
|
||||||
|
<span class="sd"> When the user selects a locale with territory (and a language), the</span>
|
||||||
|
<span class="sd"> territory has priority over the language. If any of the official languages</span>
|
||||||
|
<span class="sd"> in the territory is supported by the engine (``engine_locales``) it will</span>
|
||||||
|
<span class="sd"> be used.</span>
|
||||||
|
|
||||||
|
<span class="sd"> Second approximation rule (*by language*):</span>
|
||||||
|
|
||||||
|
<span class="sd"> If "First approximation rule" brings no result or the user selects only a</span>
|
||||||
|
<span class="sd"> language without a territory. Check in which territories the language</span>
|
||||||
|
<span class="sd"> has an official status and if one of these territories is supported by the</span>
|
||||||
|
<span class="sd"> engine.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches, too-many-return-statements</span>
|
||||||
|
|
||||||
|
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="c1"># There was a 1:1 mapping (e.g. a region "fr-BE --> fr_BE" or a language</span>
|
||||||
|
<span class="c1"># "zh --> zh"), no need to narrow language-script nor territory.</span>
|
||||||
|
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||||
|
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s1">'-'</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">Locale</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">searxng_locale</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'-'</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="k">except</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">UnknownLocaleError</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">default</span>
|
||||||
|
|
||||||
|
<span class="n">searxng_lang</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_lang</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="c1"># There was a 1:1 mapping (e.g. "zh-HK --> zh_Hant" or "zh-CN --> zh_Hans")</span>
|
||||||
|
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||||
|
|
||||||
|
<span class="c1"># SearXNG's selected locale is not supported by the engine ..</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Try to narrow by *official* languages in the territory (??-XX).</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">official_language</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">languages</span><span class="o">.</span><span class="n">get_official_languages</span><span class="p">(</span><span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">,</span> <span class="n">de_facto</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">official_language</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span>
|
||||||
|
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||||
|
|
||||||
|
<span class="c1"># Engine does not support one of the official languages in the territory or</span>
|
||||||
|
<span class="c1"># there is only a language selected without a territory.</span>
|
||||||
|
|
||||||
|
<span class="c1"># Now lets have a look if the searxng_lang (the language selected by the</span>
|
||||||
|
<span class="c1"># user) is a official language in other territories. If so, check if</span>
|
||||||
|
<span class="c1"># engine does support the searxng_lang in this other territory.</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">terr_lang_dict</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">territory</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">langs</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]</span>
|
||||||
|
<span class="k">for</span> <span class="n">territory</span><span class="p">,</span> <span class="n">langs</span> <span class="ow">in</span> <span class="n">babel</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">get_global</span><span class="p">(</span><span class="s2">"territory_languages"</span><span class="p">)</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">_lang</span> <span class="o">=</span> <span class="n">langs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_lang</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">_lang</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">_lang</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'official_status'</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">terr_lang_dict</span><span class="p">[</span><span class="n">territory</span><span class="p">]</span> <span class="o">=</span> <span class="n">_lang</span>
|
||||||
|
|
||||||
|
<span class="c1"># first: check fr-FR, de-DE .. is supported by the engine</span>
|
||||||
|
<span class="c1"># exception: 'en' --> 'en-US'</span>
|
||||||
|
|
||||||
|
<span class="n">territory</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">territory</span> <span class="o">==</span> <span class="s1">'EN'</span><span class="p">:</span>
|
||||||
|
<span class="n">territory</span> <span class="o">=</span> <span class="s1">'US'</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">terr_lang_dict</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">territory</span><span class="p">):</span>
|
||||||
|
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">territory</span>
|
||||||
|
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||||
|
|
||||||
|
<span class="c1"># second: sort by population_percent and take first match</span>
|
||||||
|
|
||||||
|
<span class="c1"># drawback of "population percent": if there is a territory with a</span>
|
||||||
|
<span class="c1"># small number of people (e.g 100) but the majority speaks the</span>
|
||||||
|
<span class="c1"># language, then the percentage might be 100% (--> 100 people) but in</span>
|
||||||
|
<span class="c1"># a different territory with more people (e.g. 10.000) where only 10%</span>
|
||||||
|
<span class="c1"># speak the language the total amount of speaker is higher (--> 200</span>
|
||||||
|
<span class="c1"># people).</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># By example: The population of Saint-Martin is 33.000, of which 100%</span>
|
||||||
|
<span class="c1"># speak French, but this is less than the 30% of the approximately 2.5</span>
|
||||||
|
<span class="c1"># million Belgian citizens</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># - 'fr-MF', 'population_percent': 100.0, 'official_status': 'official'</span>
|
||||||
|
<span class="c1"># - 'fr-BE', 'population_percent': 38.0, 'official_status': 'official'</span>
|
||||||
|
|
||||||
|
<span class="n">terr_lang_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">terr_lang_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">terr_lang_list</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">territory</span><span class="p">,</span> <span class="n">_lang</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">terr_lang_list</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">item</span><span class="p">:</span> <span class="n">item</span><span class="p">[</span><span class="mi">1</span><span class="p">][</span><span class="s1">'population_percent'</span><span class="p">],</span> <span class="n">reverse</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">locale</span><span class="o">.</span><span class="n">language</span> <span class="o">+</span> <span class="s1">'-'</span> <span class="o">+</span> <span class="n">territory</span>
|
||||||
|
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">engine_locales</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">engine_locale</span>
|
||||||
|
|
||||||
|
<span class="c1"># No luck: narrow by "language from territory" and "territory from language"</span>
|
||||||
|
<span class="c1"># does not fit to a locale supported by the engine.</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">engine_locale</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_locale</span> <span class="o">=</span> <span class="n">default</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">default</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="match_locale">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.match_locale">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">match_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">locale_tag_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">fallback</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Return tag from ``locale_tag_list`` that best fits to ``searxng_locale``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param str searxng_locale: SearXNG's internal representation of locale (de,</span>
|
||||||
|
<span class="sd"> de-DE, fr-BE, zh, zh-CN, zh-TW ..).</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param list locale_tag_list: The list of locale tags to select from</span>
|
||||||
|
|
||||||
|
<span class="sd"> :param str fallback: fallback locale tag (if unset --> ``None``)</span>
|
||||||
|
|
||||||
|
<span class="sd"> The rules to find a match are implemented in :py:obj:`get_engine_locale`,</span>
|
||||||
|
<span class="sd"> the ``engine_locales`` is build up by :py:obj:`build_engine_locales`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> The *SearXNG locale* string and the members of ``locale_tag_list`` has to</span>
|
||||||
|
<span class="sd"> be known by babel! The :py:obj:`ADDITIONAL_TRANSLATIONS` are used in the</span>
|
||||||
|
<span class="sd"> UI and are not known by babel --> will be ignored.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="c1"># searxng_locale = 'es'</span>
|
||||||
|
<span class="c1"># locale_tag_list = ['es-AR', 'es-ES', 'es-MX']</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">searxng_locale</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">fallback</span>
|
||||||
|
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">get_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">fallback</span>
|
||||||
|
|
||||||
|
<span class="c1"># normalize to a SearXNG locale that can be passed to get_engine_locale</span>
|
||||||
|
|
||||||
|
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
|
||||||
|
<span class="n">searxng_locale</span> <span class="o">=</span> <span class="n">region_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># clean up locale_tag_list</span>
|
||||||
|
|
||||||
|
<span class="n">tag_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="k">for</span> <span class="n">tag</span> <span class="ow">in</span> <span class="n">locale_tag_list</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">tag</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'all'</span><span class="p">,</span> <span class="s1">'auto'</span><span class="p">)</span> <span class="ow">or</span> <span class="n">tag</span> <span class="ow">in</span> <span class="n">ADDITIONAL_TRANSLATIONS</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">tag_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># emulate fetch_traits</span>
|
||||||
|
<span class="n">engine_locales</span> <span class="o">=</span> <span class="n">build_engine_locales</span><span class="p">(</span><span class="n">tag_list</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">get_engine_locale</span><span class="p">(</span><span class="n">searxng_locale</span><span class="p">,</span> <span class="n">engine_locales</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="n">fallback</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="build_engine_locales">
|
||||||
|
<a class="viewcode-back" href="../../src/searx.locales.html#searx.locales.build_engine_locales">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">build_engine_locales</span><span class="p">(</span><span class="n">tag_list</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""From a list of locale tags a dictionary is build that can be passed by</span>
|
||||||
|
<span class="sd"> argument ``engine_locales`` to :py:obj:`get_engine_locale`. This function</span>
|
||||||
|
<span class="sd"> is mainly used by :py:obj:`match_locale` and is similar to what the</span>
|
||||||
|
<span class="sd"> ``fetch_traits(..)`` function of engines do.</span>
|
||||||
|
|
||||||
|
<span class="sd"> If there are territory codes in the ``tag_list`` that have a *script code*</span>
|
||||||
|
<span class="sd"> additional keys are added to the returned dictionary.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> >>> import locales</span>
|
||||||
|
<span class="sd"> >>> engine_locales = locales.build_engine_locales(['en', 'en-US', 'zh', 'zh-CN', 'zh-TW'])</span>
|
||||||
|
<span class="sd"> >>> engine_locales</span>
|
||||||
|
<span class="sd"> {</span>
|
||||||
|
<span class="sd"> 'en': 'en', 'en-US': 'en-US',</span>
|
||||||
|
<span class="sd"> 'zh': 'zh', 'zh-CN': 'zh-CN', 'zh_Hans': 'zh-CN',</span>
|
||||||
|
<span class="sd"> 'zh-TW': 'zh-TW', 'zh_Hant': 'zh-TW'</span>
|
||||||
|
<span class="sd"> }</span>
|
||||||
|
<span class="sd"> >>> get_engine_locale('zh-Hans', engine_locales)</span>
|
||||||
|
<span class="sd"> 'zh-CN'</span>
|
||||||
|
|
||||||
|
<span class="sd"> This function is a good example to understand the language/region model</span>
|
||||||
|
<span class="sd"> of SearXNG:</span>
|
||||||
|
|
||||||
|
<span class="sd"> SearXNG only distinguishes between **search languages** and **search</span>
|
||||||
|
<span class="sd"> regions**, by adding the *script-tags*, languages with *script-tags* can</span>
|
||||||
|
<span class="sd"> be assigned to the **regions** that SearXNG supports.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">engine_locales</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">tag</span> <span class="ow">in</span> <span class="n">tag_list</span><span class="p">:</span>
|
||||||
|
<span class="n">locale</span> <span class="o">=</span> <span class="n">get_locale</span><span class="p">(</span><span class="n">tag</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"build_engine_locales: skip locale tag </span><span class="si">%s</span><span class="s2"> / unknown by babel"</span><span class="p">,</span> <span class="n">tag</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">territory</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_locales</span><span class="p">[</span><span class="n">region_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tag</span>
|
||||||
|
<span class="k">if</span> <span class="n">locale</span><span class="o">.</span><span class="n">script</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_locales</span><span class="p">[</span><span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tag</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">engine_locales</span><span class="p">[</span><span class="n">language_tag</span><span class="p">(</span><span class="n">locale</span><span class="p">)]</span> <span class="o">=</span> <span class="n">tag</span>
|
||||||
|
<span class="k">return</span> <span class="n">engine_locales</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
446
_modules/searx/plugins/_core.html
Normal file
@@ -0,0 +1,446 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins._core — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins._core</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins._core</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods,missing-module-docstring</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"PluginInfo"</span><span class="p">,</span> <span class="s2">"Plugin"</span><span class="p">,</span> <span class="s2">"PluginCfg"</span><span class="p">,</span> <span class="s2">"PluginStorage"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">importlib</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">inspect</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Generator</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">dataclasses</span><span class="w"> </span><span class="kn">import</span> <span class="n">dataclass</span><span class="p">,</span> <span class="n">field</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">Result</span><span class="p">,</span> <span class="n">EngineResults</span><span class="p">,</span> <span class="n">LegacyResult</span> <span class="c1"># pyright: ignore[reportPrivateLocalImportUsage]</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
|
||||||
|
|
||||||
|
<span class="n">log</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"searx.plugins"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="PluginInfo">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginInfo">[docs]</a>
|
||||||
|
<span class="nd">@dataclass</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">PluginInfo</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Object that holds information about a *plugin*, these infos are shown to</span>
|
||||||
|
<span class="sd"> the user in the Preferences menu.</span>
|
||||||
|
|
||||||
|
<span class="sd"> To be able to translate the information into other languages, the text must</span>
|
||||||
|
<span class="sd"> be written in English and translated with :py:obj:`flask_babel.gettext`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The ID-selector in HTML/CSS `#<id>`."""</span>
|
||||||
|
|
||||||
|
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the *plugin*."""</span>
|
||||||
|
|
||||||
|
<span class="n">description</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Short description of the *answerer*."""</span>
|
||||||
|
|
||||||
|
<span class="n">preference_section</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"general"</span><span class="p">,</span> <span class="s2">"ui"</span><span class="p">,</span> <span class="s2">"privacy"</span><span class="p">,</span> <span class="s2">"query"</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="s2">"general"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Section (tab/group) in the preferences where this plugin is shown to the</span>
|
||||||
|
<span class="sd"> user.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The value ``query`` is reserved for plugins that are activated via a</span>
|
||||||
|
<span class="sd"> *keyword* as part of a search query, see:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - :py:obj:`PluginInfo.examples`</span>
|
||||||
|
<span class="sd"> - :py:obj:`Plugin.keywords`</span>
|
||||||
|
|
||||||
|
<span class="sd"> Those plugins are shown in the preferences in tab *Special Queries*.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">examples</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">list</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of short examples of the usage / of query terms."""</span>
|
||||||
|
|
||||||
|
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="nb">list</span><span class="p">)</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""See :py:obj:`Plugin.keywords`"""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">ID_REGXP</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s2">"[a-z][a-z0-9].*"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Plugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Plugin</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Abstract base class of all Plugins."""</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The ID (suffix) in the HTML form."""</span>
|
||||||
|
|
||||||
|
<span class="n">active</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">ClassVar</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Plugin is enabled/disabled by default (:py:obj:`PluginCfg.active`)."""</span>
|
||||||
|
|
||||||
|
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Keywords in the search query that activate the plugin. The *keyword* is</span>
|
||||||
|
<span class="sd"> the first word in a search query. If a plugin should be executed regardless</span>
|
||||||
|
<span class="sd"> of the search query, the list of keywords should be empty (which is also the</span>
|
||||||
|
<span class="sd"> default in the base class for Plugins)."""</span>
|
||||||
|
|
||||||
|
<span class="n">log</span><span class="p">:</span> <span class="n">logging</span><span class="o">.</span><span class="n">Logger</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A logger object, is automatically initialized when calling the</span>
|
||||||
|
<span class="sd"> constructor (if not already set in the subclass)."""</span>
|
||||||
|
|
||||||
|
<span class="n">info</span><span class="p">:</span> <span class="n">PluginInfo</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Information about the *plugin*, see :py:obj:`PluginInfo`."""</span>
|
||||||
|
|
||||||
|
<span class="n">fqn</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">fqn</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">fqn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__mro__</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="vm">__module__</span>
|
||||||
|
|
||||||
|
<span class="c1"># names from the configuration</span>
|
||||||
|
<span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">plg_cfg</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># names that must be set by the plugin implementation</span>
|
||||||
|
<span class="k">for</span> <span class="n">attr</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="s2">"id"</span><span class="p">,</span>
|
||||||
|
<span class="p">]:</span>
|
||||||
|
<span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attr</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"plugin </span><span class="si">{</span><span class="bp">self</span><span class="si">}</span><span class="s2"> is missing attribute </span><span class="si">{</span><span class="n">attr</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ID_REGXP</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"plugin ID </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2"> contains invalid character (use lowercase ASCII)"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">"log"</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="n">pkg_name</span> <span class="o">=</span> <span class="n">inspect</span><span class="o">.</span><span class="n">getmodule</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">)</span><span class="o">.</span><span class="n">__package__</span> <span class="c1"># pyright: ignore[reportOptionalMemberAccess]</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">pkg_name</span><span class="si">}</span><span class="s2">.</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The hash value is used in :py:obj:`set`, for example, when an object</span>
|
||||||
|
<span class="sd"> is added to the set. The hash value is also used in other contexts,</span>
|
||||||
|
<span class="sd"> e.g. when checking for equality to identify identical plugins from</span>
|
||||||
|
<span class="sd"> different sources (name collisions)."""</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""py:obj:`Plugin` objects are equal if the hash values of the two</span>
|
||||||
|
<span class="sd"> objects are equal."""</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">hash</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Plugin.init">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">app</span><span class="p">:</span> <span class="s2">"flask.Flask"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Initialization of the plugin, the return value decides whether this</span>
|
||||||
|
<span class="sd"> plugin is active or not. Initialization only takes place once, at the</span>
|
||||||
|
<span class="sd"> time the WEB application is set up. The base method always returns</span>
|
||||||
|
<span class="sd"> ``True``, the method can be overwritten in the inheritances,</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``True`` plugin is active</span>
|
||||||
|
<span class="sd"> - ``False`` plugin is inactive</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<div class="viewcode-block" id="Plugin.pre_search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin.pre_search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">pre_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Runs BEFORE the search request and returns a boolean:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``True`` to continue the search</span>
|
||||||
|
<span class="sd"> - ``False`` to stop the search</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Plugin.on_result">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin.on_result">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">on_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="s2">"Result"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Runs for each result of each engine and returns a boolean:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``True`` to keep the result</span>
|
||||||
|
<span class="sd"> - ``False`` to remove the result from the result list</span>
|
||||||
|
|
||||||
|
<span class="sd"> The ``result`` can be modified to the needs.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. hint::</span>
|
||||||
|
|
||||||
|
<span class="sd"> If :py:obj:`Result.url <searx.result_types._base.Result.url>` is modified,</span>
|
||||||
|
<span class="sd"> :py:obj:`Result.parsed_url <searx.result_types._base.Result.parsed_url>` must</span>
|
||||||
|
<span class="sd"> be changed accordingly:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> result["parsed_url"] = urlparse(result["url"])</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Plugin.post_search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.Plugin.post_search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span>
|
||||||
|
<span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span>
|
||||||
|
<span class="p">)</span> <span class="o">-></span> <span class="s2">"None | list[Result | LegacyResult] | EngineResults"</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Runs AFTER the search request. Can return a list of</span>
|
||||||
|
<span class="sd"> :py:obj:`Result <searx.result_types._base.Result>` objects to be added to the</span>
|
||||||
|
<span class="sd"> final result list."""</span>
|
||||||
|
<span class="k">return</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="PluginCfg">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginCfg">[docs]</a>
|
||||||
|
<span class="nd">@dataclass</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">PluginCfg</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Settings of a plugin.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> mypackage.mymodule.MyPlugin:</span>
|
||||||
|
<span class="sd"> active: true</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">active</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Plugin is active by default and the user can *opt-out* in the preferences."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="PluginStorage">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">PluginStorage</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A storage for managing the *plugins* of SearXNG."""</span>
|
||||||
|
|
||||||
|
<span class="n">plugin_list</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="n">Plugin</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The list of :py:obj:`Plugins` in this storage."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">Generator</span><span class="p">[</span><span class="n">Plugin</span><span class="p">]:</span>
|
||||||
|
<span class="k">yield from</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="nd">@property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">info</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">PluginInfo</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="p">[</span><span class="n">p</span><span class="o">.</span><span class="n">info</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="PluginStorage.load_settings">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage.load_settings">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">load_settings</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Load plugins configured in SearXNG's settings :ref:`settings</span>
|
||||||
|
<span class="sd"> plugins`."""</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">fqn</span><span class="p">,</span> <span class="n">plg_settings</span> <span class="ow">in</span> <span class="n">cfg</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="bp">cls</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="n">mod_name</span><span class="p">,</span> <span class="n">cls_name</span> <span class="o">=</span> <span class="n">fqn</span><span class="o">.</span><span class="n">rsplit</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">mod</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">mod_name</span><span class="p">)</span>
|
||||||
|
<span class="bp">cls</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">cls_name</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-exception-caught</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="n">exc</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">cls</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"plugin </span><span class="si">{</span><span class="n">fqn</span><span class="si">}</span><span class="s2"> is not implemented"</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||||
|
<span class="n">plg</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">PluginCfg</span><span class="p">(</span><span class="o">**</span><span class="n">plg_settings</span><span class="p">))</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">plg</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="PluginStorage.register">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage.register">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plugin</span><span class="p">:</span> <span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Register a :py:obj:`Plugin`. In case of name collision (if two</span>
|
||||||
|
<span class="sd"> plugins have same ID) a :py:obj:`KeyError` exception is raised.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">plugin</span> <span class="ow">in</span> <span class="p">[</span><span class="n">p</span><span class="o">.</span><span class="n">id</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="p">]:</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"name collision '</span><span class="si">{</span><span class="n">plugin</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">'"</span>
|
||||||
|
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">plugin</span><span class="p">)</span>
|
||||||
|
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"plugin has been loaded"</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="PluginStorage.init">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">app</span><span class="p">:</span> <span class="s2">"flask.Flask"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Calls the method :py:obj:`Plugin.init` of each plugin in this</span>
|
||||||
|
<span class="sd"> storage. Depending on its return value, the plugin is removed from</span>
|
||||||
|
<span class="sd"> *this* storage or not."""</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">plg</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="o">.</span><span class="n">copy</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">plg</span><span class="o">.</span><span class="n">init</span><span class="p">(</span><span class="n">app</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">plg</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">pre_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">ret</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="k">for</span> <span class="n">plugin</span> <span class="ow">in</span> <span class="p">[</span><span class="n">p</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span> <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">id</span> <span class="ow">in</span> <span class="n">search</span><span class="o">.</span><span class="n">user_plugins</span><span class="p">]:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">ret</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="n">plugin</span><span class="o">.</span><span class="n">pre_search</span><span class="p">(</span><span class="n">request</span><span class="o">=</span><span class="n">request</span><span class="p">,</span> <span class="n">search</span><span class="o">=</span><span class="n">search</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||||
|
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">"Exception while calling pre_search"</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ret</span><span class="p">:</span>
|
||||||
|
<span class="c1"># skip this search on the first False from a plugin</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
<span class="k">return</span> <span class="n">ret</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">on_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="s2">"Result"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">ret</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="k">for</span> <span class="n">plugin</span> <span class="ow">in</span> <span class="p">[</span><span class="n">p</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span> <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">id</span> <span class="ow">in</span> <span class="n">search</span><span class="o">.</span><span class="n">user_plugins</span><span class="p">]:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">ret</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="n">plugin</span><span class="o">.</span><span class="n">on_result</span><span class="p">(</span><span class="n">request</span><span class="o">=</span><span class="n">request</span><span class="p">,</span> <span class="n">search</span><span class="o">=</span><span class="n">search</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||||
|
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">"Exception while calling on_result"</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ret</span><span class="p">:</span>
|
||||||
|
<span class="c1"># ignore this result item on the first False from a plugin</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">ret</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="PluginStorage.post_search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/development.html#searx.plugins.PluginStorage.post_search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="n">SXNG_Request</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Extend :py:obj:`search.result_container</span>
|
||||||
|
<span class="sd"> <searx.results.ResultContainer`> with result items from plugins listed</span>
|
||||||
|
<span class="sd"> in :py:obj:`search.user_plugins <SearchWithPlugins.user_plugins>`.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">keyword</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">for</span> <span class="n">keyword</span> <span class="ow">in</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="n">keyword</span><span class="p">:</span>
|
||||||
|
<span class="k">break</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">plugin</span> <span class="ow">in</span> <span class="p">[</span><span class="n">p</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_list</span> <span class="k">if</span> <span class="n">p</span><span class="o">.</span><span class="n">id</span> <span class="ow">in</span> <span class="n">search</span><span class="o">.</span><span class="n">user_plugins</span><span class="p">]:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">plugin</span><span class="o">.</span><span class="n">keywords</span><span class="p">:</span>
|
||||||
|
<span class="c1"># plugin with keywords: skip plugin if no keyword match</span>
|
||||||
|
<span class="k">if</span> <span class="n">keyword</span> <span class="ow">and</span> <span class="n">keyword</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">plugin</span><span class="o">.</span><span class="n">keywords</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">plugin</span><span class="o">.</span><span class="n">post_search</span><span class="p">(</span><span class="n">request</span><span class="o">=</span><span class="n">request</span><span class="p">,</span> <span class="n">search</span><span class="o">=</span><span class="n">search</span><span class="p">)</span> <span class="ow">or</span> <span class="p">[]</span>
|
||||||
|
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||||||
|
<span class="n">plugin</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">"Exception while calling post_search"</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="c1"># In case of *plugins* prefix ``plugin:`` is set, see searx.result_types.Result</span>
|
||||||
|
<span class="n">search</span><span class="o">.</span><span class="n">result_container</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="sa">f</span><span class="s2">"plugin: </span><span class="si">{</span><span class="n">plugin</span><span class="o">.</span><span class="n">id</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> <span class="n">results</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
135
_modules/searx/plugins/calculator.html
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins.calculator — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins.calculator</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins.calculator</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/calculator.html#searx.plugins.calculator.SXNGPlugin">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Parses and solves mathematical expressions."""</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span> <span class="o">=</span> <span class="s2">"calculator"</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
|
||||||
|
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Calculator"</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Parses and solves mathematical expressions."</span><span class="p">),</span>
|
||||||
|
<span class="n">preference_section</span><span class="o">=</span><span class="s2">"query"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
177
_modules/searx/plugins/hash_plugin.html
Normal file
@@ -0,0 +1,177 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins.hash_plugin — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins.hash_plugin</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins.hash_plugin</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring, missing-class-docstring</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/hash_plugin.html#searx.plugins.hash_plugin.SXNGPlugin">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Plugin converts strings to different hash digests. The results are</span>
|
||||||
|
<span class="sd"> displayed in area for the "answers".</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span> <span class="o">=</span> <span class="s2">"hash_plugin"</span>
|
||||||
|
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"md5"</span><span class="p">,</span> <span class="s2">"sha1"</span><span class="p">,</span> <span class="s2">"sha224"</span><span class="p">,</span> <span class="s2">"sha256"</span><span class="p">,</span> <span class="s2">"sha384"</span><span class="p">,</span> <span class="s2">"sha512"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">parser_re</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">f</span><span class="s2">"(</span><span class="si">{</span><span class="s1">'|'</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">)</span><span class="si">}</span><span class="s2">) (.*)"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">I</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
|
||||||
|
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Hash plugin"</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span>
|
||||||
|
<span class="s2">"Converts strings to different hash digests. Available functions: md5, sha1, sha224, sha256, sha384, sha512."</span> <span class="c1"># pylint:disable=line-too-long</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">examples</span><span class="o">=</span><span class="p">[</span><span class="s2">"sha512 The quick brown fox jumps over the lazy dog"</span><span class="p">],</span>
|
||||||
|
<span class="n">preference_section</span><span class="o">=</span><span class="s2">"query"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin.post_search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/hash_plugin.html#searx.plugins.hash_plugin.SXNGPlugin.post_search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">"SXNG_Request"</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a result list only for the first page."""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">m</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parser_re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">m</span><span class="p">:</span>
|
||||||
|
<span class="c1"># wrong query</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">function</span><span class="p">,</span> <span class="n">string</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">groups</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">string</span><span class="o">.</span><span class="n">strip</span><span class="p">():</span>
|
||||||
|
<span class="c1"># end if the string is empty</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="c1"># select hash function</span>
|
||||||
|
<span class="n">f</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="n">function</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
||||||
|
|
||||||
|
<span class="c1"># make digest from the given string</span>
|
||||||
|
<span class="n">f</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">string</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
|
||||||
|
<span class="n">answer</span> <span class="o">=</span> <span class="n">function</span> <span class="o">+</span> <span class="s2">" "</span> <span class="o">+</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"hash digest"</span><span class="p">)</span> <span class="o">+</span> <span class="s2">": "</span> <span class="o">+</span> <span class="n">f</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">answer</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
316
_modules/searx/plugins/hostnames.html
Normal file
@@ -0,0 +1,316 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins.hostnames — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins.hostnames</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins.hostnames</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches, unused-argument</span>
|
||||||
|
<span class="sd">"""During the initialization phase, the plugin checks whether a ``hostnames:``</span>
|
||||||
|
<span class="sd">configuration exists. If this is not the case, the plugin is not included in the</span>
|
||||||
|
<span class="sd">PluginStorage (it is not available for selection).</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``hostnames.replace``: A **mapping** of regular expressions to hostnames to be</span>
|
||||||
|
<span class="sd"> replaced by other hostnames.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> hostnames:</span>
|
||||||
|
<span class="sd"> replace:</span>
|
||||||
|
<span class="sd"> '(.*\\.)?youtube\\.com$': 'invidious.example.com'</span>
|
||||||
|
<span class="sd"> '(.*\\.)?youtu\\.be$': 'invidious.example.com'</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``hostnames.remove``: A **list** of regular expressions of the hostnames whose</span>
|
||||||
|
<span class="sd"> results should be taken from the results list.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> hostnames:</span>
|
||||||
|
<span class="sd"> remove:</span>
|
||||||
|
<span class="sd"> - '(.*\\.)?facebook.com$'</span>
|
||||||
|
<span class="sd"> - ...</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``hostnames.high_priority``: A **list** of regular expressions for hostnames</span>
|
||||||
|
<span class="sd"> whose result should be given higher priority. The results from these hosts are</span>
|
||||||
|
<span class="sd"> arranged higher in the results list.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> hostnames:</span>
|
||||||
|
<span class="sd"> high_priority:</span>
|
||||||
|
<span class="sd"> - '(.*\\.)?wikipedia.org$'</span>
|
||||||
|
<span class="sd"> - ...</span>
|
||||||
|
|
||||||
|
<span class="sd">- ``hostnames.lower_priority``: A **list** of regular expressions for hostnames</span>
|
||||||
|
<span class="sd"> whose result should be given lower priority. The results from these hosts are</span>
|
||||||
|
<span class="sd"> arranged lower in the results list.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> hostnames:</span>
|
||||||
|
<span class="sd"> low_priority:</span>
|
||||||
|
<span class="sd"> - '(.*\\.)?google(\\..*)?$'</span>
|
||||||
|
<span class="sd"> - ...</span>
|
||||||
|
|
||||||
|
<span class="sd">If the URL matches the pattern of ``high_priority`` AND ``low_priority``, the</span>
|
||||||
|
<span class="sd">higher priority wins over the lower priority.</span>
|
||||||
|
|
||||||
|
<span class="sd">Alternatively, you can also specify a file name for the **mappings** or</span>
|
||||||
|
<span class="sd">**lists** to load these from an external file:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> hostnames:</span>
|
||||||
|
<span class="sd"> replace: 'rewrite-hosts.yml'</span>
|
||||||
|
<span class="sd"> remove:</span>
|
||||||
|
<span class="sd"> - '(.*\\.)?facebook.com$'</span>
|
||||||
|
<span class="sd"> - ...</span>
|
||||||
|
<span class="sd"> low_priority:</span>
|
||||||
|
<span class="sd"> - '(.*\\.)?google(\\..*)?$'</span>
|
||||||
|
<span class="sd"> - ...</span>
|
||||||
|
<span class="sd"> high_priority:</span>
|
||||||
|
<span class="sd"> - '(.*\\.)?wikipedia.org$'</span>
|
||||||
|
<span class="sd"> - ...</span>
|
||||||
|
|
||||||
|
<span class="sd">The ``rewrite-hosts.yml`` from the example above must be in the folder in which</span>
|
||||||
|
<span class="sd">the ``settings.yml`` file is already located (``/etc/searxng``). The file then</span>
|
||||||
|
<span class="sd">only contains the lists or the mapping tables without further information on the</span>
|
||||||
|
<span class="sd">namespaces. In the example above, this would be a mapping table that looks</span>
|
||||||
|
<span class="sd">something like this:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. code:: yaml</span>
|
||||||
|
|
||||||
|
<span class="sd"> '(.*\\.)?youtube\\.com$': 'invidious.example.com'</span>
|
||||||
|
<span class="sd"> '(.*\\.)?youtu\\.be$': 'invidious.example.com'</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlunparse</span><span class="p">,</span> <span class="n">urlparse</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">settings</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span><span class="p">,</span> <span class="n">LegacyResult</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.settings_loader</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_yaml_cfg</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._core</span><span class="w"> </span><span class="kn">import</span> <span class="n">log</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">flask</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">Result</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
|
||||||
|
|
||||||
|
<span class="n">REPLACE</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="n">re</span><span class="o">.</span><span class="n">Pattern</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
|
||||||
|
<span class="n">REMOVE</span><span class="p">:</span> <span class="nb">set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="n">HIGH</span><span class="p">:</span> <span class="nb">set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="n">LOW</span><span class="p">:</span> <span class="nb">set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/hostnames.html#searx.plugins.hostnames.SXNGPlugin">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Rewrite hostnames, remove results or prioritize them."""</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span> <span class="o">=</span> <span class="s2">"hostnames"</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
|
||||||
|
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Hostnames plugin"</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Rewrite hostnames and remove or prioritize results based on the hostname"</span><span class="p">),</span>
|
||||||
|
<span class="n">preference_section</span><span class="o">=</span><span class="s2">"general"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin.on_result">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/hostnames.html#searx.plugins.hostnames.SXNGPlugin.on_result">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">on_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">"SXNG_Request"</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="s2">"Result"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">pattern</span> <span class="ow">in</span> <span class="n">REMOVE</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="ow">and</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
|
||||||
|
<span class="c1"># if the link (parsed_url) of the result match, then remove the</span>
|
||||||
|
<span class="c1"># result from the result list, in any other case, the result</span>
|
||||||
|
<span class="c1"># remains in the list / see final "return True" below.</span>
|
||||||
|
<span class="c1"># log.debug("FIXME: remove [url/parsed_url] %s %s", pattern.pattern, result.url)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">filter_urls</span><span class="p">(</span><span class="n">filter_url_field</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">LegacyResult</span><span class="p">)):</span>
|
||||||
|
<span class="k">for</span> <span class="n">pattern</span> <span class="ow">in</span> <span class="n">LOW</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="ow">and</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="s2">"low"</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">pattern</span> <span class="ow">in</span> <span class="n">HIGH</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="ow">and</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">priority</span> <span class="o">=</span> <span class="s2">"high"</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin.init">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/hostnames.html#searx.plugins.hostnames.SXNGPlugin.init">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">app</span><span class="p">:</span> <span class="s2">"flask.Flask"</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> <span class="c1"># pylint: disable=unused-argument</span>
|
||||||
|
<span class="k">global</span> <span class="n">REPLACE</span><span class="p">,</span> <span class="n">REMOVE</span><span class="p">,</span> <span class="n">HIGH</span><span class="p">,</span> <span class="n">LOW</span> <span class="c1"># pylint: disable=global-statement</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">):</span>
|
||||||
|
<span class="c1"># Remove plugin, if there isn't a "hostnames:" setting</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="n">REPLACE</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_regular_expressions</span><span class="p">(</span><span class="s2">"replace"</span><span class="p">)</span> <span class="ow">or</span> <span class="p">{}</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">REMOVE</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_regular_expressions</span><span class="p">(</span><span class="s2">"remove"</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">set</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">HIGH</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_regular_expressions</span><span class="p">(</span><span class="s2">"high_priority"</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">set</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
<span class="n">LOW</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_load_regular_expressions</span><span class="p">(</span><span class="s2">"low_priority"</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">set</span><span class="p">()</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_load_regular_expressions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">settings_key</span><span class="p">)</span> <span class="o">-></span> <span class="nb">dict</span><span class="p">[</span><span class="n">re</span><span class="o">.</span><span class="n">Pattern</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="nb">set</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">setting_value</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">settings_key</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">setting_value</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># load external file with configuration</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">setting_value</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">setting_value</span> <span class="o">=</span> <span class="n">get_yaml_cfg</span><span class="p">(</span><span class="n">setting_value</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">setting_value</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="p">{</span><span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">r</span><span class="p">)</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">setting_value</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">setting_value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="p">{</span><span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">p</span><span class="p">):</span> <span class="n">r</span> <span class="k">for</span> <span class="p">(</span><span class="n">p</span><span class="p">,</span> <span class="n">r</span><span class="p">)</span> <span class="ow">in</span> <span class="n">setting_value</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">None</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="filter_url_field">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/hostnames.html#searx.plugins.hostnames.filter_url_field">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">filter_url_field</span><span class="p">(</span><span class="n">result</span><span class="p">:</span> <span class="s2">"Result|LegacyResult"</span><span class="p">,</span> <span class="n">field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">url_src</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span> <span class="o">|</span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns bool ``True`` to use URL unchanged (``False`` to ignore URL).</span>
|
||||||
|
<span class="sd"> If URL should be modified, the returned string is the new URL to use."""</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">url_src</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"missing a URL in field </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">field_name</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
|
||||||
|
<span class="n">url_src_parsed</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="o">=</span><span class="n">url_src</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">pattern</span> <span class="ow">in</span> <span class="n">REMOVE</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">url_src_parsed</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">pattern</span><span class="p">,</span> <span class="n">replacement</span> <span class="ow">in</span> <span class="n">REPLACE</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="n">pattern</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">url_src_parsed</span><span class="o">.</span><span class="n">netloc</span><span class="p">):</span>
|
||||||
|
<span class="n">new_url</span> <span class="o">=</span> <span class="n">url_src_parsed</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span><span class="n">netloc</span><span class="o">=</span><span class="n">pattern</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="n">replacement</span><span class="p">,</span> <span class="n">url_src_parsed</span><span class="o">.</span><span class="n">netloc</span><span class="p">))</span>
|
||||||
|
<span class="n">new_url</span> <span class="o">=</span> <span class="n">urlunparse</span><span class="p">(</span><span class="n">new_url</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="n">new_url</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="kc">True</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
135
_modules/searx/plugins/infinite_scroll.html
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins.infinite_scroll — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins.infinite_scroll</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins.infinite_scroll</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/infinite_scroll.html#searx.plugins.infinite_scroll.SXNGPlugin">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Automatically loads the next page when scrolling to bottom of the current page."""</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span> <span class="o">=</span> <span class="s2">"infiniteScroll"</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
|
||||||
|
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Infinite scroll"</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Automatically loads the next page when scrolling to bottom of the current page"</span><span class="p">),</span>
|
||||||
|
<span class="n">preference_section</span><span class="o">=</span><span class="s2">"ui"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
169
_modules/searx/plugins/self_info.html
Normal file
@@ -0,0 +1,169 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins.self_info — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins.self_info</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins.self_info</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring, missing-class-docstring</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="n">ip_address</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/self_info.html#searx.plugins.self_info.SXNGPlugin">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Simple plugin that displays information about user's request, including</span>
|
||||||
|
<span class="sd"> the IP or HTTP User-Agent. The information is displayed in area for the</span>
|
||||||
|
<span class="sd"> "answers".</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span> <span class="o">=</span> <span class="s2">"self_info"</span>
|
||||||
|
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"ip"</span><span class="p">,</span> <span class="s2">"user-agent"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">ip_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^ip"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">IGNORECASE</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">ua_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"^user-agent"</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">IGNORECASE</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
|
||||||
|
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Self Information"</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Displays your IP if the query is "ip" and your user agent if the query is "user-agent"."""</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">preference_section</span><span class="o">=</span><span class="s2">"query"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin.post_search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/self_info.html#searx.plugins.self_info.SXNGPlugin.post_search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">"SXNG_Request"</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Returns a result list only for the first page."""</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ip_regex</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="p">)</span> <span class="ow">and</span> <span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Your IP is: "</span><span class="p">)</span> <span class="o">+</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">)</span><span class="o">.</span><span class="n">compressed</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ua_regex</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="p">):</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Your user-agent is: "</span><span class="p">)</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">user_agent</span><span class="p">)))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
179
_modules/searx/plugins/time_zone.html
Normal file
@@ -0,0 +1,179 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins.time_zone — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins.time_zone</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins.time_zone</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=missing-module-docstring</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">datetime</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.weather</span><span class="w"> </span><span class="kn">import</span> <span class="n">DateTime</span><span class="p">,</span> <span class="n">GeoLocation</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">t</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/time_zone.html#searx.plugins.time_zone.SXNGPlugin">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Plugin to display the current time at different timezones (usually the</span>
|
||||||
|
<span class="sd"> query city)."""</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"time_zone"</span>
|
||||||
|
<span class="n">keywords</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"time"</span><span class="p">,</span> <span class="s2">"timezone"</span><span class="p">,</span> <span class="s2">"now"</span><span class="p">,</span> <span class="s2">"clock"</span><span class="p">,</span> <span class="s2">"timezones"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
|
||||||
|
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Timezones plugin"</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Display the current time on different time zones."</span><span class="p">),</span>
|
||||||
|
<span class="n">preference_section</span><span class="o">=</span><span class="s2">"query"</span><span class="p">,</span>
|
||||||
|
<span class="n">examples</span><span class="o">=</span><span class="p">[</span><span class="s2">"time Berlin"</span><span class="p">,</span> <span class="s2">"clock Los Angeles"</span><span class="p">],</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin.post_search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/time_zone.html#searx.plugins.time_zone.SXNGPlugin.post_search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">"SXNG_Request"</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The plugin uses the :py:obj:`searx.weather.GeoLocation` class, which</span>
|
||||||
|
<span class="sd"> is already implemented in the context of weather forecasts, to determine</span>
|
||||||
|
<span class="sd"> the time zone. The :py:obj:`searx.weather.DateTime` class is used for</span>
|
||||||
|
<span class="sd"> the localized display of date and time."""</span>
|
||||||
|
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="c1"># remove keywords from the query</span>
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span>
|
||||||
|
<span class="n">query_parts</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">part</span><span class="p">:</span> <span class="n">part</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">,</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">" "</span><span class="p">))</span>
|
||||||
|
<span class="n">search_term</span> <span class="o">=</span> <span class="s2">" "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">query_parts</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">search_term</span><span class="p">:</span>
|
||||||
|
<span class="n">date_time</span> <span class="o">=</span> <span class="n">DateTime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">())</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">date_time</span><span class="o">.</span><span class="n">l10n</span><span class="p">()))</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">geo</span> <span class="o">=</span> <span class="n">GeoLocation</span><span class="o">.</span><span class="n">by_query</span><span class="p">(</span><span class="n">search_term</span><span class="o">=</span><span class="n">search_term</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">geo</span><span class="p">:</span>
|
||||||
|
<span class="n">date_time</span> <span class="o">=</span> <span class="n">DateTime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">tz</span><span class="o">=</span><span class="n">geo</span><span class="o">.</span><span class="n">zoneinfo</span><span class="p">))</span>
|
||||||
|
<span class="n">tz_name</span> <span class="o">=</span> <span class="n">geo</span><span class="o">.</span><span class="n">timezone</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'_'</span><span class="p">,</span> <span class="s1">' '</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span>
|
||||||
|
<span class="n">answer</span><span class="o">=</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">tz_name</span><span class="si">}</span><span class="s2">:"</span> <span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">date_time</span><span class="o">.</span><span class="n">l10n</span><span class="p">()</span><span class="si">}</span><span class="s2"> (</span><span class="si">{</span><span class="n">date_time</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">'%Z'</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
188
_modules/searx/plugins/tor_check.html
Normal file
@@ -0,0 +1,188 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins.tor_check — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins.tor_check</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins.tor_check</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""A plugin to check if the ip address of the request is a Tor exit-node if the</span>
|
||||||
|
<span class="sd">user searches for ``tor-check``. It fetches the tor exit node list from</span>
|
||||||
|
<span class="sd">:py:obj:`url_exit_list` and parses all the IPs into a list, then checks if the</span>
|
||||||
|
<span class="sd">user's IP address is in it.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">ipaddress</span><span class="w"> </span><span class="kn">import</span> <span class="n">ip_address</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">httpx</span><span class="w"> </span><span class="kn">import</span> <span class="n">HTTPError</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.network</span><span class="w"> </span><span class="kn">import</span> <span class="n">get</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># Regex for exit node addresses in the list.</span>
|
||||||
|
<span class="n">reg</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(?<=ExitAddress )\S+"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">url_exit_list</span> <span class="o">=</span> <span class="s2">"https://check.torproject.org/exit-addresses"</span>
|
||||||
|
<span class="sd">"""URL to load Tor exit list from."""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/tor_check.html#searx.plugins.tor_check.SXNGPlugin">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Rewrite hostnames, remove results or prioritize them."""</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span> <span class="o">=</span> <span class="s2">"tor_check"</span>
|
||||||
|
<span class="n">keywords</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"tor-check"</span><span class="p">,</span> <span class="s2">"tor_check"</span><span class="p">,</span> <span class="s2">"torcheck"</span><span class="p">,</span> <span class="s2">"tor"</span><span class="p">,</span> <span class="s2">"tor check"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
|
||||||
|
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Tor check plugin"</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span>
|
||||||
|
<span class="s2">"This plugin checks if the address of the request is a Tor exit-node, and"</span>
|
||||||
|
<span class="s2">" informs the user if it is; like check.torproject.org, but from SearXNG."</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">preference_section</span><span class="o">=</span><span class="s2">"query"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin.post_search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/tor_check.html#searx.plugins.tor_check.SXNGPlugin.post_search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">"SXNG_Request"</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">keywords</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="c1"># Request the list of tor exit nodes.</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="n">resp</span> <span class="o">=</span> <span class="n">get</span><span class="p">(</span><span class="n">url_exit_list</span><span class="p">)</span>
|
||||||
|
<span class="n">node_list</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">reg</span><span class="p">,</span> <span class="n">resp</span><span class="o">.</span><span class="n">text</span><span class="p">)</span> <span class="c1"># type: ignore</span>
|
||||||
|
|
||||||
|
<span class="k">except</span> <span class="n">HTTPError</span><span class="p">:</span>
|
||||||
|
<span class="c1"># No answer, return error</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"Could not download the list of Tor exit-nodes from"</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">msg</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">url_exit_list</span><span class="si">}</span><span class="s2">"</span><span class="p">))</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">real_ip</span> <span class="o">=</span> <span class="n">ip_address</span><span class="p">(</span><span class="n">address</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">remote_addr</span><span class="p">))</span><span class="o">.</span><span class="n">compressed</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">real_ip</span> <span class="ow">in</span> <span class="n">node_list</span><span class="p">:</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"You are using Tor and it looks like you have the external IP address"</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">msg</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">real_ip</span><span class="si">}</span><span class="s2">"</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">msg</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"You are not using Tor and you have the external IP address"</span><span class="p">)</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">msg</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">real_ip</span><span class="si">}</span><span class="s2">"</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
260
_modules/searx/plugins/unit_converter.html
Normal file
@@ -0,0 +1,260 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.plugins.unit_converter — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.plugins.unit_converter</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.plugins.unit_converter</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""A plugin for converting measured values from one unit to another unit (a</span>
|
||||||
|
<span class="sd">unit converter).</span>
|
||||||
|
|
||||||
|
<span class="sd">The plugin looks up the symbols (given in the query term) in a list of</span>
|
||||||
|
<span class="sd">converters, each converter is one item in the list (compare</span>
|
||||||
|
<span class="sd">:py:obj:`ADDITIONAL_UNITS`). If the symbols are ambiguous, the matching units</span>
|
||||||
|
<span class="sd">of measurement are evaluated. The weighting in the evaluation results from the</span>
|
||||||
|
<span class="sd">sorting of the :py:obj:`list of unit converters<symbol_to_si>`.</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">babel.numbers</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span><span class="p">,</span> <span class="n">get_locale</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.wikidata_units</span><span class="w"> </span><span class="kn">import</span> <span class="n">symbol_to_si</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">Plugin</span><span class="p">,</span> <span class="n">PluginInfo</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.result_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">EngineResults</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">typing</span><span class="o">.</span><span class="n">TYPE_CHECKING</span><span class="p">:</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">SearchWithPlugins</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.extended_types</span><span class="w"> </span><span class="kn">import</span> <span class="n">SXNG_Request</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.plugins</span><span class="w"> </span><span class="kn">import</span> <span class="n">PluginCfg</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">CONVERT_KEYWORDS</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"in"</span><span class="p">,</span> <span class="s2">"to"</span><span class="p">,</span> <span class="s2">"as"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/unit_converter.html#searx.plugins.unit_converter.SXNGPlugin">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">SXNGPlugin</span><span class="p">(</span><span class="n">Plugin</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Convert between units. The result is displayed in area for the</span>
|
||||||
|
<span class="sd"> "answers".</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="nb">id</span> <span class="o">=</span> <span class="s2">"unit_converter"</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plg_cfg</span><span class="p">:</span> <span class="s2">"PluginCfg"</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">plg_cfg</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">PluginInfo</span><span class="p">(</span>
|
||||||
|
<span class="nb">id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||||
|
<span class="n">name</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Unit converter plugin"</span><span class="p">),</span>
|
||||||
|
<span class="n">description</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="s2">"Convert between units"</span><span class="p">),</span>
|
||||||
|
<span class="n">preference_section</span><span class="o">=</span><span class="s2">"general"</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="SXNGPlugin.post_search">
|
||||||
|
<a class="viewcode-back" href="../../../dev/plugins/unit_converter.html#searx.plugins.unit_converter.SXNGPlugin.post_search">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">post_search</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">:</span> <span class="s2">"SXNG_Request"</span><span class="p">,</span> <span class="n">search</span><span class="p">:</span> <span class="s2">"SearchWithPlugins"</span><span class="p">)</span> <span class="o">-></span> <span class="n">EngineResults</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span> <span class="o">=</span> <span class="n">EngineResults</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># only convert between units on the first page</span>
|
||||||
|
<span class="k">if</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">pageno</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="n">query</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">search_query</span><span class="o">.</span><span class="n">query</span>
|
||||||
|
<span class="n">query_parts</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">" "</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">query_parts</span><span class="p">)</span> <span class="o"><</span> <span class="mi">3</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">results</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">query_part</span> <span class="ow">in</span> <span class="n">query_parts</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">keyword</span> <span class="ow">in</span> <span class="n">CONVERT_KEYWORDS</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">query_part</span> <span class="o">==</span> <span class="n">keyword</span><span class="p">:</span>
|
||||||
|
<span class="n">from_query</span><span class="p">,</span> <span class="n">to_query</span> <span class="o">=</span> <span class="n">query</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">keyword</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">target_val</span> <span class="o">=</span> <span class="n">_parse_text_and_convert</span><span class="p">(</span><span class="n">from_query</span><span class="o">.</span><span class="n">strip</span><span class="p">(),</span> <span class="n">to_query</span><span class="o">.</span><span class="n">strip</span><span class="p">())</span>
|
||||||
|
<span class="k">if</span> <span class="n">target_val</span><span class="p">:</span>
|
||||||
|
<span class="n">results</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">results</span><span class="o">.</span><span class="n">types</span><span class="o">.</span><span class="n">Answer</span><span class="p">(</span><span class="n">answer</span><span class="o">=</span><span class="n">target_val</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="n">results</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<span class="c1"># inspired from https://stackoverflow.com/a/42475086</span>
|
||||||
|
<span class="n">RE_MEASURE</span> <span class="o">=</span> <span class="sa">r</span><span class="s1">'''</span>
|
||||||
|
<span class="s1">(?P<sign>[-+]?) # +/- or nothing for positive</span>
|
||||||
|
<span class="s1">(\s*) # separator: white space or nothing</span>
|
||||||
|
<span class="s1">(?P<number>[\d\.,]*) # number: 1,000.00 (en) or 1.000,00 (de)</span>
|
||||||
|
<span class="s1">(?P<E>[eE][-+]?\d+)? # scientific notation: e(+/-)2 (*10^2)</span>
|
||||||
|
<span class="s1">(\s*) # separator: white space or nothing</span>
|
||||||
|
<span class="s1">(?P<unit>\S+) # unit of measure</span>
|
||||||
|
<span class="s1">'''</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_text_and_convert</span><span class="p">(</span><span class="n">from_query</span><span class="p">,</span> <span class="n">to_query</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches, too-many-locals</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">from_query</span> <span class="ow">and</span> <span class="n">to_query</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">measured</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">RE_MEASURE</span><span class="p">,</span> <span class="n">from_query</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">VERBOSE</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">measured</span> <span class="ow">and</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">'number'</span><span class="p">),</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">'unit'</span><span class="p">)):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># Symbols are not unique, if there are several hits for the from-unit, then</span>
|
||||||
|
<span class="c1"># the correct one must be determined by comparing it with the to-unit</span>
|
||||||
|
<span class="c1"># https://github.com/searxng/searxng/pull/3378#issuecomment-2080974863</span>
|
||||||
|
|
||||||
|
<span class="c1"># first: collecting possible units</span>
|
||||||
|
|
||||||
|
<span class="n">source_list</span><span class="p">,</span> <span class="n">target_list</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">symbol</span><span class="p">,</span> <span class="n">si_name</span><span class="p">,</span> <span class="n">from_si</span><span class="p">,</span> <span class="n">to_si</span><span class="p">,</span> <span class="n">orig_symbol</span> <span class="ow">in</span> <span class="n">symbol_to_si</span><span class="p">():</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">symbol</span> <span class="o">==</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">'unit'</span><span class="p">):</span>
|
||||||
|
<span class="n">source_list</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">si_name</span><span class="p">,</span> <span class="n">to_si</span><span class="p">))</span>
|
||||||
|
<span class="k">if</span> <span class="n">symbol</span> <span class="o">==</span> <span class="n">to_query</span><span class="p">:</span>
|
||||||
|
<span class="n">target_list</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">si_name</span><span class="p">,</span> <span class="n">from_si</span><span class="p">,</span> <span class="n">orig_symbol</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">source_list</span> <span class="ow">and</span> <span class="n">target_list</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">source_to_si</span> <span class="o">=</span> <span class="n">target_from_si</span> <span class="o">=</span> <span class="n">target_symbol</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># second: find the right unit by comparing list of from-units with list of to-units</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">source</span> <span class="ow">in</span> <span class="n">source_list</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">target_list</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">source</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">target</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span> <span class="c1"># compare si_name</span>
|
||||||
|
<span class="n">source_to_si</span> <span class="o">=</span> <span class="n">source</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">target_from_si</span> <span class="o">=</span> <span class="n">target</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||||
|
<span class="n">target_symbol</span> <span class="o">=</span> <span class="n">target</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">source_to_si</span> <span class="ow">and</span> <span class="n">target_from_si</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="n">_locale</span> <span class="o">=</span> <span class="n">get_locale</span><span class="p">()</span> <span class="ow">or</span> <span class="s1">'en_US'</span>
|
||||||
|
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">'sign'</span><span class="p">)</span> <span class="o">+</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">'number'</span><span class="p">)</span> <span class="o">+</span> <span class="p">(</span><span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">'E'</span><span class="p">)</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">)</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">parse_decimal</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">_locale</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># convert value to SI unit</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">source_to_si</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">)):</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">*</span> <span class="n">source_to_si</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">source_to_si</span><span class="p">(</span><span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="c1"># convert value from SI unit to target unit</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">target_from_si</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">)):</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">)</span> <span class="o">*</span> <span class="n">target_from_si</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">value</span> <span class="o">=</span> <span class="n">target_from_si</span><span class="p">(</span><span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">measured</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="s1">'E'</span><span class="p">):</span>
|
||||||
|
<span class="c1"># when incoming notation is scientific, outgoing notation is scientific</span>
|
||||||
|
<span class="n">result</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">format_scientific</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">_locale</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span> <span class="o">=</span> <span class="n">babel</span><span class="o">.</span><span class="n">numbers</span><span class="o">.</span><span class="n">format_decimal</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">locale</span><span class="o">=</span><span class="n">_locale</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">'#,##0.##########;-#'</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s1">'</span><span class="si">{</span><span class="n">result</span><span class="si">}</span><span class="s1"> </span><span class="si">{</span><span class="n">target_symbol</span><span class="si">}</span><span class="s1">'</span>
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
194
_modules/searx/result_types.html
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.result_types — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.result_types</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.result_types</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Typification of the result items generated by the *engines*, *answerers* and</span>
|
||||||
|
<span class="sd">*plugins*.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. note::</span>
|
||||||
|
|
||||||
|
<span class="sd"> We are at the beginning of typing the results. Further typing will follow,</span>
|
||||||
|
<span class="sd"> but this is a very large task that we will only be able to implement</span>
|
||||||
|
<span class="sd"> gradually. For more, please read :ref:`result types`.</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
|
<span class="s2">"Result"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"MainResult"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"KeyValue"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"EngineResults"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"AnswerSet"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Answer"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Translations"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"WeatherAnswer"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Code"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"Paper"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"File"</span><span class="p">,</span>
|
||||||
|
<span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">abc</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">Result</span><span class="p">,</span> <span class="n">MainResult</span><span class="p">,</span> <span class="n">LegacyResult</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.answer</span><span class="w"> </span><span class="kn">import</span> <span class="n">AnswerSet</span><span class="p">,</span> <span class="n">Answer</span><span class="p">,</span> <span class="n">Translations</span><span class="p">,</span> <span class="n">WeatherAnswer</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.keyvalue</span><span class="w"> </span><span class="kn">import</span> <span class="n">KeyValue</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.code</span><span class="w"> </span><span class="kn">import</span> <span class="n">Code</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.paper</span><span class="w"> </span><span class="kn">import</span> <span class="n">Paper</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">.file</span><span class="w"> </span><span class="kn">import</span> <span class="n">File</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="ResultList">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/index.html#searx.result_types.ResultList">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">ResultList</span><span class="p">(</span><span class="nb">list</span><span class="p">[</span><span class="n">Result</span> <span class="o">|</span> <span class="n">LegacyResult</span><span class="p">],</span> <span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Base class of all result lists (abstract)."""</span>
|
||||||
|
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">types</span><span class="p">:</span> <span class="c1"># pylint: disable=invalid-name</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The collection of result types (which have already been</span>
|
||||||
|
<span class="sd"> implemented)."""</span>
|
||||||
|
|
||||||
|
<span class="n">Answer</span> <span class="o">=</span> <span class="n">Answer</span>
|
||||||
|
<span class="n">KeyValue</span> <span class="o">=</span> <span class="n">KeyValue</span>
|
||||||
|
<span class="n">Code</span> <span class="o">=</span> <span class="n">Code</span>
|
||||||
|
<span class="n">Paper</span> <span class="o">=</span> <span class="n">Paper</span>
|
||||||
|
<span class="n">File</span> <span class="o">=</span> <span class="n">File</span>
|
||||||
|
<span class="n">MainResult</span> <span class="o">=</span> <span class="n">MainResult</span>
|
||||||
|
<span class="n">Result</span> <span class="o">=</span> <span class="n">Result</span>
|
||||||
|
<span class="n">Translations</span> <span class="o">=</span> <span class="n">Translations</span>
|
||||||
|
<span class="n">WeatherAnswer</span> <span class="o">=</span> <span class="n">WeatherAnswer</span>
|
||||||
|
|
||||||
|
<span class="c1"># for backward compatibility</span>
|
||||||
|
<span class="n">LegacyResult</span> <span class="o">=</span> <span class="n">LegacyResult</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="c1"># pylint: disable=useless-parent-delegation</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">add</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">Result</span> <span class="o">|</span> <span class="n">LegacyResult</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Add a :py:`Result` item to the result list."""</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">result</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="EngineResults">
|
||||||
|
<a class="viewcode-back" href="../../dev/engines/index.html#searx.result_types.EngineResults">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">EngineResults</span><span class="p">(</span><span class="n">ResultList</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Result list that should be used by engine developers. For convenience,</span>
|
||||||
|
<span class="sd"> engine developers don't need to import types / see :py:obj:`ResultList.types`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> from searx.result_types import EngineResults</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> def response(resp) -> EngineResults:</span>
|
||||||
|
<span class="sd"> res = EngineResults()</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> res.add( res.types.Answer(answer="lorem ipsum ..", url="https://example.org") )</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> return res</span>
|
||||||
|
<span class="sd"> """</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../index.html">
|
||||||
|
<img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../index.html">Module code</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
704
_modules/searx/result_types/_base.html
Normal file
@@ -0,0 +1,704 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.result_types._base — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.result_types._base</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.result_types._base</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods, missing-module-docstring</span>
|
||||||
|
<span class="sd">"""Basic types for the typification of results.</span>
|
||||||
|
|
||||||
|
<span class="sd">- :py:obj:`Result` base class</span>
|
||||||
|
<span class="sd">- :py:obj:`LegacyResult` for internal use only</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: Result</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _LegacyResult:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: LegacyResult</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"Result"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib.parse</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">warnings</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">datetime</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span> <span class="k">as</span> <span class="n">log</span>
|
||||||
|
|
||||||
|
<span class="n">WHITESPACE_REGEX</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s1">'( |</span><span class="se">\t</span><span class="s1">|</span><span class="se">\n</span><span class="s1">)+'</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">M</span> <span class="o">|</span> <span class="n">re</span><span class="o">.</span><span class="n">U</span><span class="p">)</span>
|
||||||
|
<span class="n">UNKNOWN</span> <span class="o">=</span> <span class="nb">object</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_normalize_url_fields</span><span class="p">(</span><span class="n">result</span><span class="p">:</span> <span class="s2">"Result | LegacyResult"</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="c1"># As soon we need LegacyResult not any longer, we can move this function to</span>
|
||||||
|
<span class="c1"># method Result.normalize_result_fields</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">url</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">'result: invalid URL: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span>
|
||||||
|
<span class="c1"># if the result has no scheme, use http as default</span>
|
||||||
|
<span class="n">scheme</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">scheme</span> <span class="ow">or</span> <span class="s2">"http"</span><span class="p">,</span>
|
||||||
|
<span class="n">path</span><span class="o">=</span><span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">path</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span><span class="o">.</span><span class="n">geturl</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">LegacyResult</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"infobox"</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
|
||||||
|
<span class="c1"># As soon we have InfoboxResult, we can move this function to method</span>
|
||||||
|
<span class="c1"># InfoboxResult.normalize_result_fields</span>
|
||||||
|
|
||||||
|
<span class="n">infobox_urls</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"urls"</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">infobox_urls</span><span class="p">:</span>
|
||||||
|
<span class="n">_url</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"url"</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">_url</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">_url</span><span class="p">)</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="n">_url</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span>
|
||||||
|
<span class="n">scheme</span><span class="o">=</span><span class="n">_url</span><span class="o">.</span><span class="n">scheme</span> <span class="ow">or</span> <span class="s2">"http"</span><span class="p">,</span>
|
||||||
|
<span class="c1"># netloc=_url.netloc.replace("www.", ""),</span>
|
||||||
|
<span class="n">path</span><span class="o">=</span><span class="n">_url</span><span class="o">.</span><span class="n">path</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span><span class="o">.</span><span class="n">geturl</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">infobox_id</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"id"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">infobox_id</span><span class="p">:</span>
|
||||||
|
<span class="n">_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">infobox_id</span><span class="p">)</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">id</span> <span class="o">=</span> <span class="n">_url</span><span class="o">.</span><span class="n">_replace</span><span class="p">(</span>
|
||||||
|
<span class="n">scheme</span><span class="o">=</span><span class="n">_url</span><span class="o">.</span><span class="n">scheme</span> <span class="ow">or</span> <span class="s2">"http"</span><span class="p">,</span>
|
||||||
|
<span class="c1"># netloc=_url.netloc.replace("www.", ""),</span>
|
||||||
|
<span class="n">path</span><span class="o">=</span><span class="n">_url</span><span class="o">.</span><span class="n">path</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span><span class="o">.</span><span class="n">geturl</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_normalize_text_fields</span><span class="p">(</span><span class="n">result</span><span class="p">:</span> <span class="s2">"MainResult | LegacyResult"</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="c1"># As soon we need LegacyResult not any longer, we can move this function to</span>
|
||||||
|
<span class="c1"># method MainResult.normalize_result_fields</span>
|
||||||
|
|
||||||
|
<span class="c1"># Actually, a type check should not be necessary if the engine is</span>
|
||||||
|
<span class="c1"># implemented correctly. Historically, however, we have always had a type</span>
|
||||||
|
<span class="c1"># check here.</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">title</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">title</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"result: invalid type of field 'title': </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">title</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">result</span><span class="o">.</span><span class="n">content</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"result: invalid type of field 'content': </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">))</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># normalize title and content</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">title</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">title</span> <span class="o">=</span> <span class="n">WHITESPACE_REGEX</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">" "</span><span class="p">,</span> <span class="n">result</span><span class="o">.</span><span class="n">title</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">content</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="o">=</span> <span class="n">WHITESPACE_REGEX</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">" "</span><span class="p">,</span> <span class="n">result</span><span class="o">.</span><span class="n">content</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="o">==</span> <span class="n">result</span><span class="o">.</span><span class="n">title</span><span class="p">:</span>
|
||||||
|
<span class="c1"># avoid duplicate content between the content and title fields</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">content</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_filter_urls</span><span class="p">(</span>
|
||||||
|
<span class="n">result</span><span class="p">:</span> <span class="s2">"Result | LegacyResult"</span><span class="p">,</span> <span class="n">filter_func</span><span class="p">:</span> <span class="s2">"Callable[[Result | LegacyResult, str, str], str | bool]"</span>
|
||||||
|
<span class="p">):</span>
|
||||||
|
<span class="c1"># pylint: disable=too-many-branches, too-many-statements</span>
|
||||||
|
|
||||||
|
<span class="c1"># As soon we need LegacyResult not any longer, we can move this function to</span>
|
||||||
|
<span class="c1"># method Result.</span>
|
||||||
|
|
||||||
|
<span class="n">url_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"url"</span><span class="p">,</span> <span class="s2">"iframe_src"</span><span class="p">,</span> <span class="s2">"audio_src"</span><span class="p">,</span> <span class="s2">"img_src"</span><span class="p">,</span> <span class="s2">"thumbnail_src"</span><span class="p">,</span> <span class="s2">"thumbnail"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="n">url_src</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">field_name</span> <span class="ow">in</span> <span class="n">url_fields</span><span class="p">:</span>
|
||||||
|
<span class="n">url_src</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">url_src</span><span class="p">:</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">new_url</span> <span class="o">=</span> <span class="n">filter_func</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
|
||||||
|
<span class="c1"># log.debug("filter_urls: filter_func(result, %s) '%s' -> '%s'", field_name, field_value, new_url)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">new_url</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
|
||||||
|
<span class="c1"># log.debug("filter_urls: unchanged field %s URL %s", field_name, field_value)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"filter_urls: drop field </span><span class="si">%s</span><span class="s2"> URL </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
|
||||||
|
<span class="n">new_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"filter_urls: modify field </span><span class="si">%s</span><span class="s2"> URL </span><span class="si">%s</span><span class="s2"> -> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">url_src</span><span class="p">,</span> <span class="n">new_url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">new_url</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">field_name</span> <span class="o">==</span> <span class="s2">"url"</span><span class="p">:</span>
|
||||||
|
<span class="c1"># sync parsed_url with new_url</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">new_url</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">new_url</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlparse</span><span class="p">(</span><span class="n">new_url</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># "urls": are from infobox</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># As soon we have InfoboxResult, we can move this function to method</span>
|
||||||
|
<span class="c1"># InfoboxResult.normalize_result_fields</span>
|
||||||
|
|
||||||
|
<span class="n">infobox_urls</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"urls"</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">infobox_urls</span><span class="p">:</span>
|
||||||
|
<span class="c1"># log.debug("filter_urls: infobox_urls .. %s", infobox_urls)</span>
|
||||||
|
<span class="n">new_infobox_urls</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">infobox_urls</span><span class="p">:</span>
|
||||||
|
<span class="n">url_src</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"url"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">url_src</span><span class="p">:</span>
|
||||||
|
<span class="n">new_infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">new_url</span> <span class="o">=</span> <span class="n">filter_func</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"infobox_urls"</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">new_url</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
|
||||||
|
<span class="n">new_infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="c1"># log.debug("filter_urls: leave URL in field 'urls' ('infobox_urls') unchanged -> %s", _url)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"filter_urls: remove URL from field 'urls' ('infobox_urls') URL </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
|
||||||
|
<span class="n">new_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"filter_urls: modify URL from field 'urls' ('infobox_urls') URL </span><span class="si">%s</span><span class="s2"> -> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url_src</span><span class="p">,</span> <span class="n">new_url</span><span class="p">)</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_url</span>
|
||||||
|
<span class="n">new_infobox_urls</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"urls"</span><span class="p">,</span> <span class="n">new_infobox_urls</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># "attributes": are from infobox</span>
|
||||||
|
<span class="c1">#</span>
|
||||||
|
<span class="c1"># The infobox has additional subsections for attributes, urls and relatedTopics:</span>
|
||||||
|
|
||||||
|
<span class="n">infobox_attributes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"attributes"</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">infobox_attributes</span><span class="p">:</span>
|
||||||
|
<span class="c1"># log.debug("filter_urls: infobox_attributes .. %s", infobox_attributes)</span>
|
||||||
|
<span class="n">new_infobox_attributes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]]]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">infobox_attributes</span><span class="p">:</span>
|
||||||
|
<span class="n">image</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"image"</span><span class="p">,</span> <span class="p">{})</span>
|
||||||
|
<span class="n">url_src</span> <span class="o">=</span> <span class="n">image</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"src"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">url_src</span><span class="p">:</span>
|
||||||
|
<span class="n">new_infobox_attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="n">new_url</span> <span class="o">=</span> <span class="n">filter_func</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"infobox_attributes"</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">new_url</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
|
||||||
|
<span class="n">new_infobox_attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
<span class="c1"># log.debug("filter_urls: leave URL in field 'image.src' unchanged -> %s", url_src)</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"filter_urls: drop field 'image.src' ('infobox_attributes') URL </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url_src</span><span class="p">)</span>
|
||||||
|
<span class="n">new_url</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">new_url</span><span class="p">:</span>
|
||||||
|
<span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||||
|
<span class="s2">"filter_urls: modify 'image.src' ('infobox_attributes') URL </span><span class="si">%s</span><span class="s2"> -> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
|
||||||
|
<span class="n">url_src</span><span class="p">,</span>
|
||||||
|
<span class="n">new_url</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">item</span><span class="p">[</span><span class="s2">"image"</span><span class="p">][</span><span class="s2">"src"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_url</span>
|
||||||
|
<span class="n">new_infobox_attributes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="nb">setattr</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="s2">"attributes"</span><span class="p">,</span> <span class="n">new_infobox_attributes</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">normalize_result_fields</span><span class="p">()</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_normalize_date_fields</span><span class="p">(</span><span class="n">result</span><span class="p">:</span> <span class="s2">"MainResult | LegacyResult"</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">result</span><span class="o">.</span><span class="n">publishedDate</span><span class="p">:</span> <span class="c1"># do not try to get a date from an empty string or a None type</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span> <span class="c1"># test if publishedDate >= 1900 (datetime module bug)</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">pubdate</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">publishedDate</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">'%Y-%m-</span><span class="si">%d</span><span class="s1"> %H:%M:%S%z'</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
|
||||||
|
<span class="n">result</span><span class="o">.</span><span class="n">publishedDate</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Result">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.Result">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Result</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Base class of all result types :ref:`result types`."""</span>
|
||||||
|
|
||||||
|
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A link related to this *result*"""</span>
|
||||||
|
|
||||||
|
<span class="n">engine</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the engine *this* result comes from. In case of *plugins* a</span>
|
||||||
|
<span class="sd"> prefix ``plugin:`` is set, in case of *answerer* prefix ``answerer:`` is</span>
|
||||||
|
<span class="sd"> set.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The field is optional and is initialized from the context if necessary.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">parsed_url</span><span class="p">:</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">ParseResult</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">""":py:obj:`urllib.parse.ParseResult` of :py:obj:`Result.url`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The field is optional and is initialized from the context if necessary.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Result.normalize_result_fields">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.Result.normalize_result_fields">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">normalize_result_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Normalize fields ``url`` and ``parse_sql``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - If field ``url`` is set and field ``parse_url`` is unset, init</span>
|
||||||
|
<span class="sd"> ``parse_url`` from field ``url``. The ``url`` field is initialized</span>
|
||||||
|
<span class="sd"> with the resulting value in ``parse_url``, if ``url`` and</span>
|
||||||
|
<span class="sd"> ``parse_url`` are not equal.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">_normalize_url_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Result.filter_urls">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.Result.filter_urls">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">filter_urls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filter_func</span><span class="p">:</span> <span class="s2">"Callable[[Result | LegacyResult, str, str], str | bool]"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A filter function is passed in the ``filter_func`` argument to</span>
|
||||||
|
<span class="sd"> filter and/or modify the URLs.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The filter function receives the :py:obj:`result object <Result>` as</span>
|
||||||
|
<span class="sd"> the first argument and the field name (``str``) in the second argument.</span>
|
||||||
|
<span class="sd"> In the third argument the URL string value is passed to the filter function.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The filter function is applied to all fields that contain a URL,</span>
|
||||||
|
<span class="sd"> in addition to the familiar ``url`` field, these include fields such as::</span>
|
||||||
|
|
||||||
|
<span class="sd"> ["url", "iframe_src", "audio_src", "img_src", "thumbnail_src", "thumbnail"]</span>
|
||||||
|
|
||||||
|
<span class="sd"> and the ``urls`` list of items of the infobox.</span>
|
||||||
|
|
||||||
|
<span class="sd"> For each field, the filter function is called and returns a bool or a</span>
|
||||||
|
<span class="sd"> string value:</span>
|
||||||
|
|
||||||
|
<span class="sd"> - ``True``: leave URL in field unchanged</span>
|
||||||
|
<span class="sd"> - ``False``: remove URL field from result (or remove entire result)</span>
|
||||||
|
<span class="sd"> - ``str``: modified URL to be used instead</span>
|
||||||
|
|
||||||
|
<span class="sd"> See :ref:`filter urls example`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">_filter_urls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filter_func</span><span class="o">=</span><span class="n">filter_func</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Generates a hash value that uniquely identifies the content of *this*</span>
|
||||||
|
<span class="sd"> result. The method can be adapted in the inheritance to compare results</span>
|
||||||
|
<span class="sd"> from different sources.</span>
|
||||||
|
|
||||||
|
<span class="sd"> If two result objects are not identical but have the same content, their</span>
|
||||||
|
<span class="sd"> hash values should also be identical.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The hash value is used in contexts, e.g. when checking for equality to</span>
|
||||||
|
<span class="sd"> identify identical results from different sources (engines).</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="nb">object</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""py:obj:`Result` objects are equal if the hash values of the two</span>
|
||||||
|
<span class="sd"> objects are equal. If needed, its recommended to overwrite</span>
|
||||||
|
<span class="sd"> "py:obj:`Result.__hash__`."""</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">hash</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># for legacy code where a result is treated as a Python dict</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__setitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">value</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">field_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">__struct_fields__</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">field_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__struct_fields__</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">as_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="p">{</span><span class="n">f</span><span class="p">:</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">__struct_fields__</span><span class="p">}</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Result.defaults_from">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.Result.defaults_from">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">defaults_from</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="s2">"Result"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Fields not set in *self* will be updated from the field values of the</span>
|
||||||
|
<span class="sd"> *other*.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">for</span> <span class="n">field_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">__struct_fields__</span><span class="p">:</span>
|
||||||
|
<span class="n">self_val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="n">other_val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="n">self_val</span><span class="p">:</span>
|
||||||
|
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">other_val</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="MainResult">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/main/mainresult.html#searx.result_types._base.MainResult">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">MainResult</span><span class="p">(</span><span class="n">Result</span><span class="p">):</span> <span class="c1"># pylint: disable=missing-class-docstring</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Base class of all result types displayed in :ref:`area main results`."""</span>
|
||||||
|
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"default.html"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the template used to render the result.</span>
|
||||||
|
|
||||||
|
<span class="sd"> By default :origin:`result_templates/default.html</span>
|
||||||
|
<span class="sd"> <searx/templates/simple/result_templates/default.html>` is used.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Link title of the result item."""</span>
|
||||||
|
|
||||||
|
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Extract or description of the result item"""</span>
|
||||||
|
|
||||||
|
<span class="n">img_src</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL of a image that is displayed in the result item."""</span>
|
||||||
|
|
||||||
|
<span class="n">iframe_src</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL of an embedded ``<iframe>`` / the frame is collapsible."""</span>
|
||||||
|
|
||||||
|
<span class="n">audio_src</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL of an embedded ``<audio controls>``."""</span>
|
||||||
|
|
||||||
|
<span class="n">thumbnail</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL of a thumbnail that is displayed in the result item."""</span>
|
||||||
|
|
||||||
|
<span class="n">publishedDate</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The date on which the object was published."""</span>
|
||||||
|
|
||||||
|
<span class="n">pubdate</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""String representation of :py:obj:`MainResult.publishedDate`</span>
|
||||||
|
|
||||||
|
<span class="sd"> Deprecated: it is still partially used in the templates, but will one day be</span>
|
||||||
|
<span class="sd"> completely eliminated.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">length</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Playing duration in seconds."""</span>
|
||||||
|
|
||||||
|
<span class="n">views</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""View count in humanized number format."""</span>
|
||||||
|
|
||||||
|
<span class="n">author</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Author of the title."""</span>
|
||||||
|
|
||||||
|
<span class="n">metadata</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Miscellaneous metadata."""</span>
|
||||||
|
|
||||||
|
<span class="n">PriorityType</span> <span class="o">=</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">""</span><span class="p">,</span> <span class="s2">"high"</span><span class="p">,</span> <span class="s2">"low"</span><span class="p">]</span> <span class="c1"># pyright: ignore[reportUnannotatedClassAttribute]</span>
|
||||||
|
<span class="n">priority</span><span class="p">:</span> <span class="s2">"MainResult.PriorityType"</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The priority can be set via :ref:`hostnames plugin`, for example."""</span>
|
||||||
|
|
||||||
|
<span class="n">engines</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""In a merged results list, the names of the engines that found this result</span>
|
||||||
|
<span class="sd"> are listed in this field."""</span>
|
||||||
|
|
||||||
|
<span class="c1"># open_group and close_group should not manged in the Result</span>
|
||||||
|
<span class="c1"># class (we should drop it from here!)</span>
|
||||||
|
<span class="n">open_group</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">close_group</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="n">positions</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="n">score</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<span class="n">category</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Ordinary url-results are equal if their values for</span>
|
||||||
|
<span class="sd"> :py:obj:`Result.template`, :py:obj:`Result.parsed_url` (without scheme)</span>
|
||||||
|
<span class="sd"> and :py:obj:`MainResult.img_src` are equal.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">parsed_url</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"missing a value in field 'parsed_url': </span><span class="si">{</span><span class="bp">self</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parsed_url</span>
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="sa">f</span><span class="s2">"|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">netloc</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">path</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">params</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">query</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">fragment</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="sa">f</span><span class="s2">"|</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">img_src</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="MainResult.normalize_result_fields">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/main/mainresult.html#searx.result_types._base.MainResult.normalize_result_fields">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">normalize_result_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">normalize_result_fields</span><span class="p">()</span>
|
||||||
|
<span class="n">_normalize_text_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||||
|
<span class="n">_normalize_date_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="LegacyResult">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.LegacyResult">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">LegacyResult</span><span class="p">(</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">]):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A wrapper around a legacy result item. The SearXNG core uses this class</span>
|
||||||
|
<span class="sd"> for untyped dictionaries / to be downward compatible.</span>
|
||||||
|
|
||||||
|
<span class="sd"> This class is needed until we have implemented an :py:obj:`Result` class for</span>
|
||||||
|
<span class="sd"> each result type and the old usages in the codebase have been fully</span>
|
||||||
|
<span class="sd"> migrated.</span>
|
||||||
|
|
||||||
|
<span class="sd"> There is only one place where this class is used, in the</span>
|
||||||
|
<span class="sd"> :py:obj:`searx.results.ResultContainer`.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. attention::</span>
|
||||||
|
|
||||||
|
<span class="sd"> Do not use this class in your own implementations!</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">UNSET</span><span class="p">:</span> <span class="nb">object</span> <span class="o">=</span> <span class="nb">object</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># emulate field types from type class Result</span>
|
||||||
|
<span class="n">url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span>
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">engine</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">parsed_url</span><span class="p">:</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">ParseResult</span> <span class="o">|</span> <span class="kc">None</span>
|
||||||
|
|
||||||
|
<span class="c1"># emulate field types from type class MainResult</span>
|
||||||
|
<span class="n">title</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">img_src</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">thumbnail</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">priority</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Literal</span><span class="p">[</span><span class="s2">""</span><span class="p">,</span> <span class="s2">"high"</span><span class="p">,</span> <span class="s2">"low"</span><span class="p">]</span>
|
||||||
|
<span class="n">engines</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
|
||||||
|
<span class="n">positions</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span>
|
||||||
|
<span class="n">score</span><span class="p">:</span> <span class="nb">float</span>
|
||||||
|
<span class="n">category</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="n">publishedDate</span><span class="p">:</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span> <span class="o">|</span> <span class="kc">None</span>
|
||||||
|
<span class="n">pubdate</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="c1"># infobox result</span>
|
||||||
|
<span class="n">urls</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span>
|
||||||
|
<span class="n">attributes</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">as_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># emulate field types from type class Result</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"url"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"url"</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"template"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"template"</span><span class="p">,</span> <span class="s2">"default.html"</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"engine"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"engine"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"parsed_url"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"parsed_url"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># emulate field types from type class MainResult</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"title"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"title"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"content"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"content"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"img_src"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"img_src"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"thumbnail"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"thumbnail"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"priority"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"priority"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"engines"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"engines"</span><span class="p">,</span> <span class="nb">set</span><span class="p">())</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"positions"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"positions"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"score"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"score"</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"category"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"category"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"publishedDate"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"publishedDate"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s2">"infobox"</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"urls"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"urls"</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="s2">"attributes"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"attributes"</span><span class="p">,</span> <span class="p">[])</span>
|
||||||
|
|
||||||
|
<span class="c1"># Legacy types that have already been ported to a type ..</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s2">"answer"</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
|
||||||
|
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"engine </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="si">}</span><span class="s2"> is using deprecated `dict` for answers"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" / use a class from searx.result_types.answer"</span><span class="p">,</span>
|
||||||
|
<span class="ne">DeprecationWarning</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">=</span> <span class="s2">"answer/legacy.html"</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">==</span> <span class="s2">"keyvalue.html"</span><span class="p">:</span>
|
||||||
|
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"engine </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="si">}</span><span class="s2"> is using deprecated `dict` for key/value results"</span>
|
||||||
|
<span class="sa">f</span><span class="s2">" / use a class from searx.result_types"</span><span class="p">,</span>
|
||||||
|
<span class="ne">DeprecationWarning</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">default</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span> <span class="o">=</span> <span class="n">UNSET</span><span class="p">)</span> <span class="o">-></span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">default</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">UNSET</span> <span class="ow">and</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"LegacyResult object has no field named: </span><span class="si">{</span><span class="n">name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
<span class="k">return</span> <span class="bp">self</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">val</span><span class="p">:</span> <span class="n">t</span><span class="o">.</span><span class="n">Any</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="n">name</span><span class="p">]</span> <span class="o">=</span> <span class="n">val</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> <span class="c1"># pyright: ignore[reportIncompatibleVariableOverride]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="s2">"answer"</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
|
||||||
|
<span class="c1"># deprecated ..</span>
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="p">[</span><span class="s2">"answer"</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">template</span> <span class="o">==</span> <span class="s2">"images.html"</span><span class="p">:</span>
|
||||||
|
<span class="c1"># image results are equal if their values for template, the url and</span>
|
||||||
|
<span class="c1"># the img_src are equal.</span>
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">img_src</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="bp">cls</span> <span class="ow">in</span> <span class="bp">self</span> <span class="k">for</span> <span class="bp">cls</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"suggestion"</span><span class="p">,</span> <span class="s2">"correction"</span><span class="p">,</span> <span class="s2">"infobox"</span><span class="p">,</span> <span class="s2">"number_of_results"</span><span class="p">,</span> <span class="s2">"engine_data"</span><span class="p">]):</span>
|
||||||
|
<span class="c1"># Ordinary url-results are equal if their values for template,</span>
|
||||||
|
<span class="c1"># parsed_url (without schema) and img_src` are equal.</span>
|
||||||
|
|
||||||
|
<span class="c1"># Code copied from with MainResult.__hash__:</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">parsed_url</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"missing a value in field 'parsed_url': </span><span class="si">{</span><span class="bp">self</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parsed_url</span>
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span>
|
||||||
|
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">template</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="sa">f</span><span class="s2">"|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">netloc</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">path</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">params</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">query</span><span class="si">}</span><span class="s2">|</span><span class="si">{</span><span class="n">url</span><span class="o">.</span><span class="n">fragment</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="o">+</span> <span class="sa">f</span><span class="s2">"|</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">img_src</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="nb">object</span><span class="p">):</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">hash</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="k">return</span> <span class="sa">f</span><span class="s2">"LegacyResult: </span><span class="si">{</span><span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__repr__</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">normalize_result_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="n">_normalize_date_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||||
|
<span class="n">_normalize_url_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||||
|
<span class="n">_normalize_text_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">engines</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">engine</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">defaults_from</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">:</span> <span class="s2">"LegacyResult"</span><span class="p">):</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">other</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="LegacyResult.filter_urls">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/base_result.html#searx.result_types._base.LegacyResult.filter_urls">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">filter_urls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filter_func</span><span class="p">:</span> <span class="s2">"Callable[[Result | LegacyResult, str, str], str | bool]"</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""See :py:obj:`Result.filter_urls`"""</span>
|
||||||
|
<span class="n">_filter_urls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filter_func</span><span class="o">=</span><span class="n">filter_func</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../result_types.html">searx.result_types</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
366
_modules/searx/result_types/answer.html
Normal file
@@ -0,0 +1,366 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.result_types.answer — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.result_types.answer</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.result_types.answer</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Typification of the *answer* results. Results of this type are rendered in</span>
|
||||||
|
<span class="sd">the :origin:`answers.html <searx/templates/simple/elements/answers.html>`</span>
|
||||||
|
<span class="sd">template.</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: BaseAnswer</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: Answer</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: Translations</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: WeatherAnswer</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: AnswerSet</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"AnswerSet"</span><span class="p">,</span> <span class="s2">"Answer"</span><span class="p">,</span> <span class="s2">"Translations"</span><span class="p">,</span> <span class="s2">"WeatherAnswer"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">flask_babel</span><span class="w"> </span><span class="kn">import</span> <span class="n">gettext</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">msgspec</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx</span><span class="w"> </span><span class="kn">import</span> <span class="n">weather</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">Result</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="BaseAnswer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.BaseAnswer">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">BaseAnswer</span><span class="p">(</span><span class="n">Result</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Base class of all answer types. It is not intended to build instances of</span>
|
||||||
|
<span class="sd"> this class (aka *abstract*)."""</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="AnswerSet">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.AnswerSet">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">AnswerSet</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Aggregator for :py:obj:`BaseAnswer` items in a result container."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__bool__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">add</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">answer</span><span class="p">:</span> <span class="n">BaseAnswer</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="n">a_hash</span> <span class="o">=</span> <span class="nb">hash</span><span class="p">(</span><span class="n">answer</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="nb">hash</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="o">==</span> <span class="n">a_hash</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">answer</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__iter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Sort items in this set and iterate over the items."""</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">answer</span><span class="p">:</span> <span class="n">answer</span><span class="o">.</span><span class="n">template</span><span class="p">)</span>
|
||||||
|
<span class="k">yield from</span> <span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__contains__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">answer</span><span class="p">:</span> <span class="n">BaseAnswer</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="n">a_hash</span> <span class="o">=</span> <span class="nb">hash</span><span class="p">(</span><span class="n">answer</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_answerlist</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="nb">hash</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="o">==</span> <span class="n">a_hash</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="kc">True</span>
|
||||||
|
<span class="k">return</span> <span class="kc">False</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Answer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.Answer">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Answer</span><span class="p">(</span><span class="n">BaseAnswer</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Simple answer type where the *answer* is a simple string with an optional</span>
|
||||||
|
<span class="sd"> :py:obj:`url field <Result.url>` field to link a resource (article, map, ..)</span>
|
||||||
|
<span class="sd"> related to the answer."""</span>
|
||||||
|
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"answer/legacy.html"</span>
|
||||||
|
|
||||||
|
<span class="n">answer</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Text of the answer."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The hash value of field *answer* is the hash value of the</span>
|
||||||
|
<span class="sd"> :py:obj:`Answer` object. :py:obj:`Answer <Result.__eq__>` objects are</span>
|
||||||
|
<span class="sd"> equal, when the hash values of both objects are equal."""</span>
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">answer</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Translations">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.Translations">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Translations</span><span class="p">(</span><span class="n">BaseAnswer</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Answer type with a list of translations.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The items in the list of :py:obj:`Translations.translations` are of type</span>
|
||||||
|
<span class="sd"> :py:obj:`Translations.Item`:</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. code:: python</span>
|
||||||
|
|
||||||
|
<span class="sd"> def response(resp):</span>
|
||||||
|
<span class="sd"> results = []</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> foo_1 = Translations.Item(</span>
|
||||||
|
<span class="sd"> text="foobar",</span>
|
||||||
|
<span class="sd"> synonyms=["bar", "foo"],</span>
|
||||||
|
<span class="sd"> examples=["foo and bar are placeholders"],</span>
|
||||||
|
<span class="sd"> )</span>
|
||||||
|
<span class="sd"> foo_url="https://www.deepl.com/de/translator#en/de/foo"</span>
|
||||||
|
<span class="sd"> ...</span>
|
||||||
|
<span class="sd"> Translations(results=results, translations=[foo], url=foo_url)</span>
|
||||||
|
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"answer/translations.html"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The template in :origin:`answer/translations.html</span>
|
||||||
|
<span class="sd"> <searx/templates/simple/answer/translations.html>`"""</span>
|
||||||
|
|
||||||
|
<span class="n">translations</span><span class="p">:</span> <span class="s2">"list[Translations.Item]"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of translations."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">translations</span><span class="p">:</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Translation does not have an item in the list translations"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Translations.Item">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.Translations.Item">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Item</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A single element of the translations / a translation. A translation</span>
|
||||||
|
<span class="sd"> consists of at least a mandatory ``text`` property (the translation) ,</span>
|
||||||
|
<span class="sd"> optional properties such as *definitions*, *synonyms* and *examples* are</span>
|
||||||
|
<span class="sd"> possible."""</span>
|
||||||
|
|
||||||
|
<span class="n">text</span><span class="p">:</span> <span class="nb">str</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Translated text."""</span>
|
||||||
|
|
||||||
|
<span class="n">transliteration</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Transliteration_ of the requested translation.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _Transliteration: https://en.wikipedia.org/wiki/Transliteration</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">examples</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of examples for the requested translation."""</span>
|
||||||
|
|
||||||
|
<span class="n">definitions</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of definitions for the requested translation."""</span>
|
||||||
|
|
||||||
|
<span class="n">synonyms</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of synonyms for the requested translation."""</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="WeatherAnswer">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.WeatherAnswer">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">WeatherAnswer</span><span class="p">(</span><span class="n">BaseAnswer</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Answer type for weather data."""</span>
|
||||||
|
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"answer/weather.html"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The template is located at :origin:`answer/weather.html</span>
|
||||||
|
<span class="sd"> <searx/templates/simple/answer/weather.html>`"""</span>
|
||||||
|
|
||||||
|
<span class="n">current</span><span class="p">:</span> <span class="s2">"WeatherAnswer.Item"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Current weather at ``location``."""</span>
|
||||||
|
|
||||||
|
<span class="n">forecasts</span><span class="p">:</span> <span class="s2">"list[WeatherAnswer.Item]"</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Weather forecasts for ``location``."""</span>
|
||||||
|
|
||||||
|
<span class="n">service</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Weather service from which this information was provided."""</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="WeatherAnswer.Item">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/answer.html#searx.result_types.answer.WeatherAnswer.Item">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Item</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Weather parameters valid for a specific point in time."""</span>
|
||||||
|
|
||||||
|
<span class="n">location</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">GeoLocation</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The geo-location the weather data is from (e.g. `Berlin, Germany`)."""</span>
|
||||||
|
|
||||||
|
<span class="n">temperature</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">Temperature</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Air temperature at 2m above the ground."""</span>
|
||||||
|
|
||||||
|
<span class="n">condition</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">WeatherConditionType</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Standardized designations that summarize the weather situation</span>
|
||||||
|
<span class="sd"> (e.g. ``light sleet showers and thunder``)."""</span>
|
||||||
|
|
||||||
|
<span class="c1"># optional fields</span>
|
||||||
|
|
||||||
|
<span class="n">datetime</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">DateTime</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Time of the forecast - not needed for the current weather."""</span>
|
||||||
|
|
||||||
|
<span class="n">summary</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""One-liner about the weather forecast / current weather conditions.</span>
|
||||||
|
<span class="sd"> If unset, a summary is build up from temperature and current weather</span>
|
||||||
|
<span class="sd"> conditions.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">feels_like</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">Temperature</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Apparent temperature, the temperature equivalent perceived by</span>
|
||||||
|
<span class="sd"> humans, caused by the combined effects of air temperature, relative</span>
|
||||||
|
<span class="sd"> humidity and wind speed. The measure is most commonly applied to the</span>
|
||||||
|
<span class="sd"> perceived outdoor temperature.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">pressure</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">Pressure</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Air pressure at sea level (e.g. 1030 hPa) """</span>
|
||||||
|
|
||||||
|
<span class="n">humidity</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">RelativeHumidity</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Amount of relative humidity in the air at 2m above the ground. The</span>
|
||||||
|
<span class="sd"> unit is ``%``, e.g. 60%)</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">wind_from</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">Compass</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The directon which moves towards / direction the wind is coming from."""</span>
|
||||||
|
|
||||||
|
<span class="n">wind_speed</span><span class="p">:</span> <span class="n">weather</span><span class="o">.</span><span class="n">WindSpeed</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Speed of wind / wind speed at 10m above the ground (10 min average)."""</span>
|
||||||
|
|
||||||
|
<span class="n">cloud_cover</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Amount of sky covered by clouds / total cloud cover for all heights</span>
|
||||||
|
<span class="sd"> (cloudiness, unit: %)"""</span>
|
||||||
|
|
||||||
|
<span class="c1"># attributes: dict[str, str | int] = {}</span>
|
||||||
|
<span class="c1"># """Key-Value dict of additional typeless weather attributes."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">summary</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">summary</span> <span class="o">=</span> <span class="n">gettext</span><span class="p">(</span><span class="s2">"</span><span class="si">{location}</span><span class="s2">: </span><span class="si">{temperature}</span><span class="s2">, </span><span class="si">{condition}</span><span class="s2">"</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||||
|
<span class="n">location</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
|
||||||
|
<span class="n">temperature</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">temperature</span><span class="p">,</span>
|
||||||
|
<span class="n">condition</span><span class="o">=</span><span class="n">gettext</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">condition</span><span class="o">.</span><span class="n">capitalize</span><span class="p">()),</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="nd">@property</span>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">url</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Determines a `data URL`_ with a symbol for the weather</span>
|
||||||
|
<span class="sd"> conditions. If no symbol can be assigned, ``None`` is returned.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _data URL:</span>
|
||||||
|
<span class="sd"> https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/data</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="n">weather</span><span class="o">.</span><span class="n">symbol_url</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">condition</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../result_types.html">searx.result_types</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
294
_modules/searx/result_types/code.html
Normal file
@@ -0,0 +1,294 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.result_types.code — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.result_types.code</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.result_types.code</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Typification of the *code* results. Results of this type are rendered in</span>
|
||||||
|
<span class="sd">the :origin:`code.html <searx/templates/simple/result_templates/code.html>`</span>
|
||||||
|
<span class="sd">template. For highlighting the code passages, Pygments_ is used.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _Pygments: https://pygments.org</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: Code</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods, disable=invalid-name</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"Code"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments</span><span class="w"> </span><span class="kn">import</span> <span class="n">highlight</span> <span class="c1"># pyright: ignore[reportUnknownVariableType]</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments.lexers._mapping</span><span class="w"> </span><span class="kn">import</span> <span class="n">LEXERS</span> <span class="c1"># pyright: ignore[reportMissingTypeStubs]</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments.lexers</span><span class="w"> </span><span class="kn">import</span> <span class="n">guess_lexer</span><span class="p">,</span> <span class="n">get_lexer_by_name</span><span class="p">,</span> <span class="n">guess_lexer_for_filename</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments.util</span><span class="w"> </span><span class="kn">import</span> <span class="n">ClassNotFound</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">pygments.formatters</span><span class="w"> </span><span class="kn">import</span> <span class="n">HtmlFormatter</span> <span class="c1"># pylint: disable=no-name-in-module</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">_pygments_languages</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">is_valid_language</span><span class="p">(</span><span class="n">code_language</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Checks if the specified ``code_language`` is known in Pygments."""</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">_pygments_languages</span><span class="p">:</span>
|
||||||
|
<span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">LEXERS</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
|
||||||
|
<span class="c1"># l[2] is the tuple with the alias names</span>
|
||||||
|
<span class="k">for</span> <span class="n">alias_name</span> <span class="ow">in</span> <span class="n">l</span><span class="p">[</span><span class="mi">2</span><span class="p">]:</span>
|
||||||
|
<span class="n">_pygments_languages</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">alias_name</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
||||||
|
<span class="k">return</span> <span class="n">code_language</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">in</span> <span class="n">_pygments_languages</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Code">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/main/code.html#searx.result_types.code.Code">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Code</span><span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Result type suitable for displaying code passages."""</span>
|
||||||
|
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"code.html"</span>
|
||||||
|
|
||||||
|
<span class="n">repository</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A link related to a repository related to the *result*."""</span>
|
||||||
|
|
||||||
|
<span class="n">codelines</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A list of two digit tuples where the first item is the line number and</span>
|
||||||
|
<span class="sd"> the second item is the code line."""</span>
|
||||||
|
|
||||||
|
<span class="n">hl_lines</span><span class="p">:</span> <span class="nb">set</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""A list of line numbers to highlight."""</span>
|
||||||
|
|
||||||
|
<span class="n">code_language</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"<guess>"</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Pygment's short name of the lexer, e.g. ``text`` for the</span>
|
||||||
|
<span class="sd"> :py:obj:`pygments.lexers.special.TextLexer`. For a list of available</span>
|
||||||
|
<span class="sd"> languages consult: `Pygments languages`_. If the language is not in this</span>
|
||||||
|
<span class="sd"> list, a :py:obj:`ValueError` is raised.</span>
|
||||||
|
|
||||||
|
<span class="sd"> The default is ``<guess>`` which has a special meaning;</span>
|
||||||
|
|
||||||
|
<span class="sd"> - If :py:obj:`Code.filename` is set, Pygment's factory method</span>
|
||||||
|
<span class="sd"> :py:obj:`pygments.lexers.guess_lexer_for_filename` is used to determine</span>
|
||||||
|
<span class="sd"> the language of the ``codelines``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> - else Pygment's :py:obj:`pygments.lexers.guess_lexer` factory is used.</span>
|
||||||
|
|
||||||
|
<span class="sd"> In case the language can't be detected, the fallback is ``text``.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _Pygments languages: https://pygments.org/languages/</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
|
||||||
|
<span class="n">filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Optional file name, can help to ``<guess>`` the language of the code (in</span>
|
||||||
|
<span class="sd"> case of ambiguous short code examples). If :py:obj:`Code.title` is not set,</span>
|
||||||
|
<span class="sd"> its default is the filename."""</span>
|
||||||
|
|
||||||
|
<span class="n">strip_new_lines</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Strip leading and trailing newlines for each returned fragment (default:</span>
|
||||||
|
<span class="sd"> ``True``). Single file might return multiple code fragments."""</span>
|
||||||
|
|
||||||
|
<span class="n">strip_whitespace</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Strip all leading and trailing whitespace for each returned fragment</span>
|
||||||
|
<span class="sd"> (default: ``False``). Single file might return multiple code fragments.</span>
|
||||||
|
<span class="sd"> Enabling this might break code indentation."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">__post_init__</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">title</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">title</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">code_language</span> <span class="o">!=</span> <span class="s2">"<guess>"</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">is_valid_language</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">code_language</span><span class="p">):</span>
|
||||||
|
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"unknown code_language: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">code_language</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The hash value is build up from URL and code lines. :py:obj:`Code</span>
|
||||||
|
<span class="sd"> <Result.__eq__>` objects are equal, when the hash values of both objects</span>
|
||||||
|
<span class="sd"> are equal.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">codelines</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">get_lexer</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">code_language</span> <span class="o">!=</span> <span class="s2">"<guess>"</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">get_lexer_by_name</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">code_language</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">src_code</span> <span class="o">=</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">l</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">codelines</span><span class="p">])</span>
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">:</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">guess_lexer_for_filename</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">,</span> <span class="n">src_code</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">ClassNotFound</span><span class="p">:</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
<span class="k">try</span><span class="p">:</span>
|
||||||
|
<span class="k">return</span> <span class="n">guess_lexer</span><span class="p">(</span><span class="n">src_code</span><span class="p">)</span>
|
||||||
|
<span class="k">except</span> <span class="n">ClassNotFound</span><span class="p">:</span>
|
||||||
|
<span class="k">pass</span>
|
||||||
|
<span class="k">return</span> <span class="n">get_lexer_by_name</span><span class="p">(</span><span class="s2">"text"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Code.HTML">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/main/code.html#searx.result_types.code.Code.HTML">[docs]</a>
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">HTML</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> <span class="c1"># pyright: ignore[reportUnknownParameterType, reportMissingParameterType]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Rendered HTML, additional options are accepted, for more details have</span>
|
||||||
|
<span class="sd"> a look at HtmlFormatter_.</span>
|
||||||
|
|
||||||
|
<span class="sd"> .. _HtmlFormatter: https://pygments.org/docs/formatters/#HtmlFormatter</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="n">lexer</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_lexer</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="n">line_no</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span> <span class="c1"># current line number</span>
|
||||||
|
<span class="n">code_block_start</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span> <span class="c1"># line where the current code block starts</span>
|
||||||
|
<span class="n">code_block_end</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># line where the current code ends</span>
|
||||||
|
<span class="n">code_block</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># lines of the current code block</span>
|
||||||
|
<span class="n">html_code_blocks</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span> <span class="c1"># HTML representation of all code blocks</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">_render</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> <span class="c1"># pyright: ignore[reportUnknownParameterType, reportMissingParameterType]</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">default</span> <span class="ow">in</span> <span class="p">[</span>
|
||||||
|
<span class="p">(</span><span class="s2">"linenos"</span><span class="p">,</span> <span class="s2">"inline"</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"linenostart"</span><span class="p">,</span> <span class="n">code_block_start</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"cssclass"</span><span class="p">,</span> <span class="s2">"code-highlight"</span><span class="p">),</span>
|
||||||
|
<span class="p">(</span><span class="s2">"hl_lines"</span><span class="p">,</span> <span class="p">[</span><span class="n">hl</span> <span class="o">-</span> <span class="n">code_block_start</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">hl</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">hl_lines</span><span class="p">]),</span>
|
||||||
|
<span class="p">]:</span>
|
||||||
|
<span class="n">kwargs</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownMemberType]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Wrap the code inside <pre> blocks using <code>, as recommended by</span>
|
||||||
|
<span class="c1"># the HTML5 specification (default is False). Do we need this?</span>
|
||||||
|
<span class="n">kwargs</span><span class="p">[</span><span class="s2">"wrapcode"</span><span class="p">]</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"wrapcode"</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">html_code_blocks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||||
|
<span class="n">highlight</span><span class="p">(</span>
|
||||||
|
<span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">code_block</span><span class="p">),</span>
|
||||||
|
<span class="n">lexer</span><span class="p">,</span>
|
||||||
|
<span class="n">HtmlFormatter</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">),</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">line_no</span><span class="p">,</span> <span class="n">code_line</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">codelines</span><span class="p">:</span>
|
||||||
|
<span class="k">if</span> <span class="n">code_block_end</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="c1"># initial start condition</span>
|
||||||
|
<span class="n">code_block_start</span> <span class="o">=</span> <span class="n">line_no</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="n">code_block_end</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">code_block_end</span> <span class="o">+</span> <span class="mi">1</span> <span class="o">!=</span> <span class="n">line_no</span><span class="p">:</span>
|
||||||
|
<span class="c1"># new code block is detected, render current code block</span>
|
||||||
|
<span class="n">_render</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
<span class="c1"># reset conditions for next code block, which first line is the</span>
|
||||||
|
<span class="c1"># current code line</span>
|
||||||
|
<span class="n">code_block</span> <span class="o">=</span> <span class="p">[</span><span class="n">code_line</span><span class="p">]</span>
|
||||||
|
<span class="n">code_block_start</span> <span class="o">=</span> <span class="n">line_no</span>
|
||||||
|
<span class="n">code_block_end</span> <span class="o">=</span> <span class="n">line_no</span>
|
||||||
|
<span class="k">continue</span>
|
||||||
|
|
||||||
|
<span class="c1"># add line to the current code block and update last line n</span>
|
||||||
|
<span class="n">code_block</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">code_line</span><span class="p">)</span>
|
||||||
|
<span class="n">code_block_end</span> <span class="o">=</span> <span class="n">line_no</span>
|
||||||
|
|
||||||
|
<span class="c1"># highlight (last) code block</span>
|
||||||
|
<span class="n">_render</span><span class="p">(</span><span class="o">**</span><span class="n">options</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportUnknownArgumentType]</span>
|
||||||
|
<span class="k">return</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">html_code_blocks</span><span class="p">)</span></div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../result_types.html">searx.result_types</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
205
_modules/searx/result_types/file.html
Normal file
@@ -0,0 +1,205 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.result_types.file — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.result_types.file</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.result_types.file</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Typification of the *file* results. Results of this type are rendered in</span>
|
||||||
|
<span class="sd">the :origin:`file.html <searx/templates/simple/result_templates/file.html>`</span>
|
||||||
|
<span class="sd">template.</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: File</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"File"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">mimetypes</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="File">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/main/file.html#searx.result_types.file.File">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">File</span><span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Class for results of type *file*"""</span>
|
||||||
|
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"file.html"</span>
|
||||||
|
|
||||||
|
<span class="n">filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the file."""</span>
|
||||||
|
|
||||||
|
<span class="n">size</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Size of bytes in human readable notation (``MB`` for 1024 * 1024 Bytes</span>
|
||||||
|
<span class="sd"> file size.)"""</span>
|
||||||
|
|
||||||
|
<span class="n">time</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Indication of a time, such as the date of the last modification or the</span>
|
||||||
|
<span class="sd"> date of creation. This is a simple string, the *date* of which can be freely</span>
|
||||||
|
<span class="sd"> chosen according to the context."""</span>
|
||||||
|
|
||||||
|
<span class="n">mimetype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Mimetype/Subtype of the file. For ``audio`` and ``video``, a URL can be</span>
|
||||||
|
<span class="sd"> passed in the :py:obj:`File.embedded` field to embed the referenced media in</span>
|
||||||
|
<span class="sd"> the result. If no value is specified, the MIME type is determined from</span>
|
||||||
|
<span class="sd"> ``self.filename`` or, alternatively, from ``self.embedded`` (if either of</span>
|
||||||
|
<span class="sd"> the two values is set)."""</span>
|
||||||
|
|
||||||
|
<span class="n">abstract</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Abstract of the file."""</span>
|
||||||
|
|
||||||
|
<span class="n">author</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Author of the file."""</span>
|
||||||
|
|
||||||
|
<span class="n">embedded</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL of an embedded media type (audio or video) / is collapsible."""</span>
|
||||||
|
|
||||||
|
<span class="n">mtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Used for displaying :py:obj:`File.embedded`. Its value is automatically</span>
|
||||||
|
<span class="sd"> populated from the base type of :py:obj:`File.mimetype`, and can be</span>
|
||||||
|
<span class="sd"> explicitly set to enforce e.g. ``audio`` or ``video`` when mimetype is</span>
|
||||||
|
<span class="sd"> something like "application/ogg" but its know the content is for example a</span>
|
||||||
|
<span class="sd"> video."""</span>
|
||||||
|
|
||||||
|
<span class="n">subtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Used for displaying :py:obj:`File.embedded`. Its value is automatically</span>
|
||||||
|
<span class="sd"> populated from the subtype type of :py:obj:`File.mimetype`, and can be</span>
|
||||||
|
<span class="sd"> explicitly set to enforce a subtype for the :py:obj:`File.embedded`</span>
|
||||||
|
<span class="sd"> element."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">__post_init__</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mtype</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">subtype</span><span class="p">:</span>
|
||||||
|
|
||||||
|
<span class="n">fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedded</span>
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mimetype</span> <span class="ow">and</span> <span class="n">fn</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">mimetype</span> <span class="o">=</span> <span class="n">mimetypes</span><span class="o">.</span><span class="n">guess_type</span><span class="p">(</span><span class="n">fn</span><span class="p">,</span> <span class="n">strict</span><span class="o">=</span><span class="kc">False</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">""</span>
|
||||||
|
|
||||||
|
<span class="n">mtype</span><span class="p">,</span> <span class="n">subtype</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mimetype</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"/"</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="p">[</span><span class="s2">""</span><span class="p">])[:</span><span class="mi">2</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">mtype</span><span class="p">:</span>
|
||||||
|
<span class="c1"># I don't know why, but the ogg video stream is not displayed,</span>
|
||||||
|
<span class="c1"># may https://github.com/videojs/video.js can help?</span>
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedded</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">".ogv"</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">mtype</span> <span class="o">=</span> <span class="s2">"video"</span>
|
||||||
|
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedded</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">".oga"</span><span class="p">):</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">mtype</span> <span class="o">=</span> <span class="s2">"audio"</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">mtype</span> <span class="o">=</span> <span class="n">mtype</span>
|
||||||
|
|
||||||
|
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">subtype</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">subtype</span> <span class="o">=</span> <span class="n">subtype</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../result_types.html">searx.result_types</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
159
_modules/searx/result_types/keyvalue.html
Normal file
@@ -0,0 +1,159 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.result_types.keyvalue — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.result_types.keyvalue</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.result_types.keyvalue</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="sd">Typification of the *keyvalue* results. Results of this type are rendered in</span>
|
||||||
|
<span class="sd">the :origin:`keyvalue.html <searx/templates/simple/result_templates/keyvalue.html>`</span>
|
||||||
|
<span class="sd">template.</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: KeyValue</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"KeyValue"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="KeyValue">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/main/keyvalue.html#searx.result_types.keyvalue.KeyValue">[docs]</a>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">KeyValue</span><span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Simple table view which maps *key* names (first col) to *values*</span>
|
||||||
|
<span class="sd"> (second col)."""</span>
|
||||||
|
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"keyvalue.html"</span>
|
||||||
|
|
||||||
|
<span class="n">kvmap</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span> <span class="o">|</span> <span class="n">OrderedDict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">typing</span><span class="o">.</span><span class="n">Any</span><span class="p">]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Dictionary with keys and values. To sort keys, use :py:obj:`OrderedDict`."""</span>
|
||||||
|
|
||||||
|
<span class="n">caption</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Optional caption for this result."""</span>
|
||||||
|
|
||||||
|
<span class="n">key_title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Optional title for the *key column*."""</span>
|
||||||
|
|
||||||
|
<span class="n">value_title</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Optional title for the *value column*."""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""The KeyValues objects are checked for object identity, even if all</span>
|
||||||
|
<span class="sd"> fields of two results have the same values, they are different from each</span>
|
||||||
|
<span class="sd"> other.</span>
|
||||||
|
<span class="sd"> """</span>
|
||||||
|
<span class="k">return</span> <span class="nb">id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../result_types.html">searx.result_types</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
205
_modules/searx/result_types/paper.html
Normal file
@@ -0,0 +1,205 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html lang="en" data-content_root="../../../">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>searx.result_types.paper — SearXNG Documentation (2026.1.16+97814c62a)</title>
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=6625fa76" />
|
||||||
|
<link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" />
|
||||||
|
<script src="../../../_static/documentation_options.js?v=5932b86d"></script>
|
||||||
|
<script src="../../../_static/doctools.js?v=9bcbadda"></script>
|
||||||
|
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||||
|
<script data-project="searxng" data-version="2026.1.16+97814c62a" src="../../../_static/describe_version.js?v=fa7f30d0"></script>
|
||||||
|
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||||
|
<link rel="search" title="Search" href="../../../search.html" />
|
||||||
|
</head><body>
|
||||||
|
<div class="related" role="navigation" aria-label="Related">
|
||||||
|
<h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li class="right" style="margin-right: 10px">
|
||||||
|
<a href="../../../genindex.html" title="General Index"
|
||||||
|
accesskey="I">index</a></li>
|
||||||
|
<li class="right" >
|
||||||
|
<a href="../../../py-modindex.html" title="Python Module Index"
|
||||||
|
>modules</a> |</li>
|
||||||
|
<li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2026.1.16+97814c62a)</a> »</li>
|
||||||
|
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
|
||||||
|
<li class="nav-item nav-item-2"><a href="../result_types.html" accesskey="U">searx.result_types</a> »</li>
|
||||||
|
<li class="nav-item nav-item-this"><a href="">searx.result_types.paper</a></li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="document">
|
||||||
|
<div class="documentwrapper">
|
||||||
|
<div class="bodywrapper">
|
||||||
|
<div class="body" role="main">
|
||||||
|
|
||||||
|
<h1>Source code for searx.result_types.paper</h1><div class="highlight"><pre>
|
||||||
|
<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span>
|
||||||
|
<span class="sd">"""Typification of the *paper* results.</span>
|
||||||
|
|
||||||
|
<span class="sd">.. _BibTeX field types: https://en.wikipedia.org/wiki/BibTeX#Field_types</span>
|
||||||
|
<span class="sd">.. _BibTeX format: https://www.bibtex.com/g/bibtex-format/</span>
|
||||||
|
|
||||||
|
<span class="sd">Results of this type are rendered in the :origin:`paper.html</span>
|
||||||
|
<span class="sd"><searx/templates/simple/result_templates/paper.html>` template.</span>
|
||||||
|
|
||||||
|
<span class="sd">Related topics:</span>
|
||||||
|
|
||||||
|
<span class="sd">- `BibTeX field types`_</span>
|
||||||
|
<span class="sd">- `BibTeX format`_</span>
|
||||||
|
|
||||||
|
<span class="sd">----</span>
|
||||||
|
|
||||||
|
<span class="sd">.. autoclass:: Paper</span>
|
||||||
|
<span class="sd"> :members:</span>
|
||||||
|
<span class="sd"> :show-inheritance:</span>
|
||||||
|
|
||||||
|
<span class="sd">"""</span>
|
||||||
|
<span class="c1"># pylint: disable=too-few-public-methods, disable=invalid-name</span>
|
||||||
|
|
||||||
|
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"Paper"</span><span class="p">]</span>
|
||||||
|
|
||||||
|
<span class="kn">import</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">t</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">searx.weather</span><span class="w"> </span><span class="kn">import</span> <span class="n">DateTime</span>
|
||||||
|
<span class="kn">from</span><span class="w"> </span><span class="nn">._base</span><span class="w"> </span><span class="kn">import</span> <span class="n">MainResult</span>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="viewcode-block" id="Paper">
|
||||||
|
<a class="viewcode-back" href="../../../dev/result_types/main/paper.html#searx.result_types.paper.Paper">[docs]</a>
|
||||||
|
<span class="nd">@t</span><span class="o">.</span><span class="n">final</span>
|
||||||
|
<span class="k">class</span><span class="w"> </span><span class="nc">Paper</span><span class="p">(</span><span class="n">MainResult</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Result type suitable for displaying scientific papers and other</span>
|
||||||
|
<span class="sd"> documents."""</span>
|
||||||
|
|
||||||
|
<span class="n">template</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"paper.html"</span>
|
||||||
|
|
||||||
|
<span class="n">date_of_publication</span><span class="p">:</span> <span class="n">DateTime</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Date the document was published."""</span>
|
||||||
|
|
||||||
|
<span class="n">content</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""An abstract or excerpt from the document."""</span>
|
||||||
|
|
||||||
|
<span class="n">comments</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Free text display in italic below the content."""</span>
|
||||||
|
|
||||||
|
<span class="n">tags</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Free tag list."""</span>
|
||||||
|
|
||||||
|
<span class="nb">type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Short description of medium type, e.g. *book*, *pdf* or *html* ..."""</span>
|
||||||
|
|
||||||
|
<span class="n">authors</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of authors of the work (authors with a "s" suffix, the "author" is</span>
|
||||||
|
<span class="sd"> in the :py:obj:`MainResult.author`)."""</span>
|
||||||
|
|
||||||
|
<span class="n">editor</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Editor of the book/paper."""</span>
|
||||||
|
|
||||||
|
<span class="n">publisher</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the publisher."""</span>
|
||||||
|
|
||||||
|
<span class="n">journal</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Name of the journal or magazine the article was published in."""</span>
|
||||||
|
|
||||||
|
<span class="n">volume</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">int</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Volume number."""</span>
|
||||||
|
|
||||||
|
<span class="n">pages</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Page range where the article is."""</span>
|
||||||
|
|
||||||
|
<span class="n">number</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""Number of the report or the issue number for a journal article."""</span>
|
||||||
|
|
||||||
|
<span class="n">doi</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""DOI number (like ``10.1038/d41586-018-07848-2``)."""</span>
|
||||||
|
|
||||||
|
<span class="n">issn</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of ISSN numbers like ``1476-4687``"""</span>
|
||||||
|
|
||||||
|
<span class="n">isbn</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""List of ISBN numbers like ``9780201896831``"""</span>
|
||||||
|
|
||||||
|
<span class="n">pdf_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL to the full article, the PDF version"""</span>
|
||||||
|
|
||||||
|
<span class="n">html_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="w"> </span><span class="sd">"""URL to full article, HTML version"""</span>
|
||||||
|
|
||||||
|
<span class="k">def</span><span class="w"> </span><span class="nf">__post_init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||||
|
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">__post_init__</span><span class="p">()</span>
|
||||||
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">date_of_publication</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">publishedDate</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<span class="bp">self</span><span class="o">.</span><span class="n">date_of_publication</span> <span class="o">=</span> <span class="n">DateTime</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">publishedDate</span><span class="p">)</span></div>
|
||||||
|
|
||||||
|
</pre></div>
|
||||||
|
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<span id="sidebar-top"></span>
|
||||||
|
<div class="sphinxsidebar" role="navigation" aria-label="Main">
|
||||||
|
<div class="sphinxsidebarwrapper">
|
||||||
|
|
||||||
|
|
||||||
|
<p class="logo"><a href="../../../index.html">
|
||||||
|
<img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/>
|
||||||
|
</a></p>
|
||||||
|
|
||||||
|
|
||||||
|
<h3><a href="../../../index.html">Table of Contents</a></h3>
|
||||||
|
<ul>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li>
|
||||||
|
<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h3>Project Links</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://github.com/searxng/searxng/tree/master">Source</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/wiki">Wiki</a>
|
||||||
|
|
||||||
|
<li><a href="https://searx.space">Public instances</a>
|
||||||
|
|
||||||
|
<li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a>
|
||||||
|
</ul><h3>Navigation</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../../index.html">Overview</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../../index.html">Module code</a>
|
||||||
|
<ul>
|
||||||
|
<li><a href="../result_types.html">searx.result_types</a>
|
||||||
|
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
<search id="searchbox" style="display: none" role="search">
|
||||||
|
<h3 id="searchlabel">Quick search</h3>
|
||||||
|
<div class="searchformwrapper">
|
||||||
|
<form class="search" action="../../../search.html" method="get">
|
||||||
|
<input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
|
||||||
|
<input type="submit" value="Go" />
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</search>
|
||||||
|
<script>document.getElementById('searchbox').style.display = "block"</script>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="clearer"></div>
|
||||||
|
</div>
|
||||||
|
<div class="footer" role="contentinfo">
|
||||||
|
© Copyright SearXNG team.
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||